From ec32e87b9ccf40f75ccc10d8e0b02bdc000f1e4f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 09:01:47 +0000 Subject: [PATCH 1/2] Update dependency kimdre/doco-cd to v0.75.0 --- .doco-cd-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.doco-cd-version b/.doco-cd-version index 9c1e7d4..4476ed3 100644 --- a/.doco-cd-version +++ b/.doco-cd-version @@ -1 +1 @@ -v0.74.0 +v0.75.0 From 82fe2209dd3dae6de737815824e60fcccb822299 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 19 Mar 2026 09:02:00 +0000 Subject: [PATCH 2/2] [Auto] Sync source code for v0.75.0 --- doco-cd-src/.github/workflows/test.yaml | 2 +- doco-cd-src/Dockerfile | 2 +- doco-cd-src/internal/config/deploy_config.go | 2 +- doco-cd-src/internal/config/utils.go | 36 +++--------- doco-cd-src/internal/docker/compose.go | 29 ++-------- doco-cd-src/internal/docker/compose_test.go | 11 ++-- doco-cd-src/internal/docker/swarm.go | 8 +-- doco-cd-src/internal/docker/swarm/loader.go | 8 +-- doco-cd-src/internal/docker/swarm_test.go | 8 +-- doco-cd-src/internal/encryption/decrypt.go | 55 ++----------------- doco-cd-src/internal/git/git.go | 7 ++- doco-cd-src/internal/stages/stage_1_init.go | 24 ++++++++ .../internal/stages/stage_2_pre-deploy.go | 33 ++++------- doco-cd-src/internal/stages/stage_3_deploy.go | 2 +- doco-cd-src/internal/stages/types.go | 4 +- doco-cd-src/test/test.compose.yaml | 2 + 16 files changed, 76 insertions(+), 157 deletions(-) diff --git a/doco-cd-src/.github/workflows/test.yaml b/doco-cd-src/.github/workflows/test.yaml index f8679e4..c5c81bd 100644 --- a/doco-cd-src/.github/workflows/test.yaml +++ b/doco-cd-src/.github/workflows/test.yaml @@ -98,7 +98,7 @@ jobs: CC: musl-gcc BW_SDK_BUILD_FLAGS: "-linkmode external -extldflags '-static -Wl,-unresolved-symbols=ignore-all'" - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage-${{ matrix.mode }}.out diff --git a/doco-cd-src/Dockerfile b/doco-cd-src/Dockerfile index f5a6735..e72c691 100644 --- a/doco-cd-src/Dockerfile +++ b/doco-cd-src/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1@sha256:4a43a54dd1fedceb30ba47e76cfcf2b47304f4161c0caeac2db1c61804ea3c91 -FROM golang:1.26.1@sha256:c7e98cc0fd4dfb71ee7465fee6c9a5f079163307e4bf141b336bb9dae00159a5 AS prerequisites +FROM golang:1.26.1@sha256:c42e4d75186af6a44eb4159dcfac758ef1c05a7011a0052fe8a8df016d8e8fb9 AS prerequisites ARG APP_VERSION=dev ARG DISABLE_BITWARDEN=false diff --git a/doco-cd-src/internal/config/deploy_config.go b/doco-cd-src/internal/config/deploy_config.go index fee126e..c4878e5 100644 --- a/doco-cd-src/internal/config/deploy_config.go +++ b/doco-cd-src/internal/config/deploy_config.go @@ -71,7 +71,7 @@ type DeployConfig struct { } `yaml:"auto_discover_opts"` // AutoDiscoverOpts are options for the autodiscovery feature Internal struct { File string `yaml:"-"` // File is the path to the deployment configuration file in the repository (if RepositoryUrl is not set) or in the cloned repository (if RepositoryUrl is set) - Environment map[string]string // Environment stores environment variables from local env_files entries (if RepositoryUrl to set) for the deployment for interpolating variables in the compose files + Environment map[string]string // Environment is stores environment variables for variable interpolation in the compose project Hash string `yaml:"-"` // Hash is a hash of the DeployConfig struct (without changing the order of its elements) } // Internal holds internal configuration values that are not set by the user } diff --git a/doco-cd-src/internal/config/utils.go b/doco-cd-src/internal/config/utils.go index 13ad7ae..e0a14d9 100644 --- a/doco-cd-src/internal/config/utils.go +++ b/doco-cd-src/internal/config/utils.go @@ -65,17 +65,20 @@ func ParseConfigFromEnv(config interface{}, mappings *[]EnvVarFileMapping) error } // LoadLocalDotEnv processes local dotenv files and loads their variables into the DeployConfig.Internal.Environment map. -func LoadLocalDotEnv(deployConfig *DeployConfig, internalRepoPath string) error { +// Remote dotenv files (prefixed with "remote:") are collected and left in DeployConfig.EnvFiles for later processing. +func LoadLocalDotEnv(deployConfig *DeployConfig, basePath string) error { const remotePrefix = "remote:" var remoteEnvFiles []string // List of env files that are not local and will be processed later - envVars := make(map[string]string) + if len(deployConfig.Internal.Environment) == 0 { + deployConfig.Internal.Environment = make(map[string]string) + } for _, f := range deployConfig.EnvFiles { // Process any env-files that are local and not in the remote repository (see repository_url) if !strings.HasPrefix(f, remotePrefix) { - absPath := filepath.Join(internalRepoPath, f) + absPath := filepath.Join(basePath, f) // Decrypt file if needed isEncrypted, err := encryption.IsEncryptedFile(absPath) @@ -108,7 +111,7 @@ func LoadLocalDotEnv(deployConfig *DeployConfig, internalRepoPath string) error } for k, v := range envMap { - envVars[k] = v + deployConfig.Internal.Environment[k] = v } } else { f = strings.TrimPrefix(f, remotePrefix) @@ -117,31 +120,6 @@ func LoadLocalDotEnv(deployConfig *DeployConfig, internalRepoPath string) error } deployConfig.EnvFiles = remoteEnvFiles - deployConfig.Internal.Environment = envVars return nil } - -// CreateTmpDotEnvFile creates a temporary dotenv file from the DeployConfig.Internal.Environment map. -func CreateTmpDotEnvFile(deployConfig *DeployConfig) (string, error) { - tmpEnvFile, err := os.CreateTemp(os.TempDir(), deployConfig.Name+".*.env") - if err != nil { - errMsg := "failed to create temporary env file" - return "", fmt.Errorf("%s: %w", errMsg, err) - } - - defer tmpEnvFile.Close() - - // Write environment variables to the temp env file - for k, v := range deployConfig.Internal.Environment { - _, err = fmt.Fprintf(tmpEnvFile, "%s=%s\n", k, v) - if err != nil { - return "", fmt.Errorf("failed to write to temporary env file: %w", err) - } - } - - // Prepend the temp env file to the list of env files - deployConfig.EnvFiles = append([]string{tmpEnvFile.Name()}, deployConfig.EnvFiles...) - - return tmpEnvFile.Name(), nil -} diff --git a/doco-cd-src/internal/docker/compose.go b/doco-cd-src/internal/docker/compose.go index 74e11ba..1a1ac5a 100644 --- a/doco-cd-src/internal/docker/compose.go +++ b/doco-cd-src/internal/docker/compose.go @@ -19,7 +19,6 @@ import ( "github.com/kimdre/doco-cd/internal/utils/module" "github.com/kimdre/doco-cd/internal/docker/swarm" - secrettypes "github.com/kimdre/doco-cd/internal/secretprovider/types" "github.com/kimdre/doco-cd/internal/utils/set" "github.com/kimdre/doco-cd/internal/utils/slice" @@ -37,7 +36,6 @@ import ( "github.com/docker/compose/v5/pkg/compose" "github.com/kimdre/doco-cd/internal/config" - "github.com/kimdre/doco-cd/internal/logger" "github.com/kimdre/doco-cd/internal/prometheus" "github.com/kimdre/doco-cd/internal/webhook" ) @@ -221,7 +219,7 @@ func LoadCompose(ctx context.Context, repoPath, workingDir, projectName string, decryptFiles := slices.Concat(absComposeFiles, absEnvFiles) for _, file := range decryptFiles { - decrypted, err := encryption.DecryptFileInPlace(file, repoPath) + decrypted, err := encryption.DecryptFileInPlace(file) if err != nil { return nil, fmt.Errorf("failed to decrypt file %s: %w", file, err) } @@ -418,7 +416,6 @@ func DeployStack( jobLog *slog.Logger, externalRepoPath string, ctx *context.Context, dockerCli *command.Cli, dockerClient *client.Client, payload *webhook.ParsedPayload, deployConfig *config.DeployConfig, changedFiles []gitInternal.ChangedFile, latestCommit, appVersion string, forceDeploy bool, - resolvedSecrets secrettypes.ResolvedSecrets, ) error { startTime := time.Now() @@ -436,24 +433,8 @@ func DeployStack( return fmt.Errorf("%s", errMsg) } - // Create a temporary env file if environment variables are specified in the deployment config - if deployConfig.Internal.Environment != nil { - tmpEnvFile, err := config.CreateTmpDotEnvFile(deployConfig) - if err != nil { - errMsg := "failed to create temporary env file" - return fmt.Errorf("%s: %w", errMsg, err) - } - - // Delete the temp file after deployment - defer func(name string) { - err = os.Remove(name) - if err != nil { - stackLog.Warn("failed to delete temporary env file", logger.ErrAttr(err), slog.String("file", name)) - } - }(tmpEnvFile) - } - - project, err := LoadCompose(*ctx, externalRepoPath, externalWorkingDir, deployConfig.Name, deployConfig.ComposeFiles, deployConfig.EnvFiles, deployConfig.Profiles, resolvedSecrets) + project, err := LoadCompose(*ctx, externalRepoPath, externalWorkingDir, deployConfig.Name, deployConfig.ComposeFiles, + deployConfig.EnvFiles, deployConfig.Profiles, deployConfig.Internal.Environment) if err != nil { return fmt.Errorf("failed to load compose config: %w", err) } @@ -488,7 +469,7 @@ func DeployStack( if swarm.ModeEnabled { stackLog.Info("deploying swarm stack") - cfg, opts, err := LoadSwarmStack(dockerCli, project, deployConfig, resolvedSecrets, externalWorkingDir) + cfg, opts, err := LoadSwarmStack(dockerCli, project, deployConfig, externalWorkingDir) if err != nil { return fmt.Errorf("failed to load swarm stack: %w", err) } @@ -1151,7 +1132,7 @@ func DecryptProjectFiles(repoPath string, p *types.Project) ([]string, error) { f = filepath.Join(p.WorkingDir, f) } - decrypted, err := encryption.DecryptFileInPlace(f, repoPath) + decrypted, err := encryption.DecryptFileInPlace(f) if err != nil { return decryptedFiles, fmt.Errorf("failed to decrypt project file '%s': %w", f, err) } diff --git a/doco-cd-src/internal/docker/compose_test.go b/doco-cd-src/internal/docker/compose_test.go index 4bfeafe..6d0afcd 100644 --- a/doco-cd-src/internal/docker/compose_test.go +++ b/doco-cd-src/internal/docker/compose_test.go @@ -21,8 +21,6 @@ import ( "github.com/kimdre/doco-cd/internal/secretprovider/bitwardensecretsmanager" - secrettypes "github.com/kimdre/doco-cd/internal/secretprovider/types" - "github.com/kimdre/doco-cd/internal/secretprovider" "github.com/kimdre/doco-cd/internal/docker/swarm" @@ -284,12 +282,15 @@ compose_files: testLog := logger.New(slog.LevelInfo) jobLog := testLog.With(slog.String("job_id", jobID)) - resolvedSecrets := make(secrettypes.ResolvedSecrets) if secretProvider != nil && len(deployConf.ExternalSecrets) > 0 { - resolvedSecrets, err = secretProvider.ResolveSecretReferences(ctx, deployConf.ExternalSecrets) + resolvedSecrets, err := secretProvider.ResolveSecretReferences(ctx, deployConf.ExternalSecrets) if err != nil { t.Fatalf("failed to resolve external secrets: %s", err.Error()) } + + for k, v := range resolvedSecrets { + deployConf.Internal.Environment[k] = v + } } err = retry.New( @@ -300,7 +301,7 @@ compose_files: }), ).Do(func() error { return DeployStack(jobLog, repoPath, &ctx, &dockerCli, dockerClient, &p, deployConf, - []git.ChangedFile{}, latestCommit, "dev", false, resolvedSecrets) + []git.ChangedFile{}, latestCommit, "dev", false) }) if err != nil { t.Fatalf("failed to deploy stack: %v", err) diff --git a/doco-cd-src/internal/docker/swarm.go b/doco-cd-src/internal/docker/swarm.go index a059813..ce6a21b 100644 --- a/doco-cd-src/internal/docker/swarm.go +++ b/doco-cd-src/internal/docker/swarm.go @@ -17,8 +17,6 @@ import ( swarmTypes "github.com/moby/moby/api/types/swarm" dockerClient "github.com/moby/moby/client" - secrettypes "github.com/kimdre/doco-cd/internal/secretprovider/types" - swarmInternal "github.com/kimdre/doco-cd/internal/docker/swarm" "github.com/compose-spec/compose-go/v2/types" @@ -38,8 +36,8 @@ var ( ) // LoadSwarmStack loads a Docker Swarm stack using the provided project and deploy configuration. -func LoadSwarmStack(dockerCli *command.Cli, project *types.Project, deployConfig *config.DeployConfig, - resolvedSecrets secrettypes.ResolvedSecrets, externalWorkingDir string, +func LoadSwarmStack(dockerCli *command.Cli, project *types.Project, + deployConfig *config.DeployConfig, externalWorkingDir string, ) (*composetypes.Config, *options.Deploy, error) { opts := options.Deploy{ Composefiles: project.ComposeFiles, @@ -51,7 +49,7 @@ func LoadSwarmStack(dockerCli *command.Cli, project *types.Project, deployConfig Environment: project.Environment, } - cfg, err := swarmInternal.LoadComposefile(*dockerCli, opts, resolvedSecrets, externalWorkingDir) + cfg, err := swarmInternal.LoadComposefile(*dockerCli, opts, deployConfig.Internal.Environment, externalWorkingDir) if err != nil { return nil, nil, fmt.Errorf("failed to load compose file: %w", err) } diff --git a/doco-cd-src/internal/docker/swarm/loader.go b/doco-cd-src/internal/docker/swarm/loader.go index 6f24dcb..458aa09 100644 --- a/doco-cd-src/internal/docker/swarm/loader.go +++ b/doco-cd-src/internal/docker/swarm/loader.go @@ -16,13 +16,11 @@ import ( "github.com/docker/cli/cli/compose/schema" "github.com/docker/cli/cli/compose/types" - secrettypes "github.com/kimdre/doco-cd/internal/secretprovider/types" - "github.com/kimdre/doco-cd/internal/docker/options" ) // LoadComposefile parse the composefile specified in the cli and returns its Config and version. -func LoadComposefile(dockerCli command.Cli, opts options.Deploy, resolvedSecrets secrettypes.ResolvedSecrets, workingDir string) (*types.Config, error) { +func LoadComposefile(dockerCli command.Cli, opts options.Deploy, environment map[string]string, workingDir string) (*types.Config, error) { configDetails, err := GetConfigDetails(opts.Composefiles, dockerCli.In()) if err != nil { return nil, err @@ -43,8 +41,8 @@ func LoadComposefile(dockerCli command.Cli, opts options.Deploy, resolvedSecrets configDetails.Environment[k] = v } - // Inject external secrets into the environment for variable interpolation - for k, v := range resolvedSecrets { + // Add additional env vars into the environment for variable interpolation + for k, v := range environment { configDetails.Environment[k] = v } diff --git a/doco-cd-src/internal/docker/swarm_test.go b/doco-cd-src/internal/docker/swarm_test.go index 15e53bd..8ff6a5c 100644 --- a/doco-cd-src/internal/docker/swarm_test.go +++ b/doco-cd-src/internal/docker/swarm_test.go @@ -10,8 +10,6 @@ import ( "github.com/moby/moby/client" "github.com/kimdre/doco-cd/internal/encryption" - secrettypes "github.com/kimdre/doco-cd/internal/secretprovider/types" - "github.com/kimdre/doco-cd/internal/test" "github.com/kimdre/doco-cd/internal/docker/swarm" @@ -81,9 +79,7 @@ func TestDeploySwarmStack(t *testing.T) { repoPath := worktree.Filesystem.Root() filePath := filepath.Join(repoPath, "docker-compose.yml") - resolvedSecrets := secrettypes.ResolvedSecrets{} - - project, err := LoadCompose(t.Context(), tmpDir, tmpDir, stackName, []string{filePath}, []string{".env"}, []string{}, resolvedSecrets) + project, err := LoadCompose(t.Context(), tmpDir, tmpDir, stackName, []string{filePath}, []string{".env"}, []string{}, map[string]string{}) if err != nil { t.Fatal(err) } @@ -95,7 +91,7 @@ func TestDeploySwarmStack(t *testing.T) { ctx := t.Context() - cfg, opts, err := LoadSwarmStack(&dockerCli, project, deployConfigs[0], resolvedSecrets, tmpDir) + cfg, opts, err := LoadSwarmStack(&dockerCli, project, deployConfigs[0], tmpDir) if err != nil { t.Fatalf("Failed to load swarm stack: %v", err) } diff --git a/doco-cd-src/internal/encryption/decrypt.go b/doco-cd-src/internal/encryption/decrypt.go index ff37810..096ea21 100644 --- a/doco-cd-src/internal/encryption/decrypt.go +++ b/doco-cd-src/internal/encryption/decrypt.go @@ -76,14 +76,7 @@ func DecryptFilesInDirectory(repoPath, dirPath string) ([]string, error) { } } - // Open the repository root for writing decrypted files without changing their permissions. - root, err := os.OpenRoot(repoPath) - if err != nil { - return nil, fmt.Errorf("failed to open repo root %s: %w", repoPath, err) - } - defer root.Close() - - err = filepath.WalkDir(dirPath, func(path string, d fs.DirEntry, err error) error { + err := filepath.WalkDir(dirPath, func(path string, d fs.DirEntry, err error) error { if err != nil { return fmt.Errorf("failed to walk directory %s: %w", path, err) } @@ -121,16 +114,6 @@ func DecryptFilesInDirectory(repoPath, dirPath string) ([]string, error) { absTarget = filepath.Join(filepath.Dir(path), target) } - // Prevent absTarget to escape the repoPath - relPath, err := filepath.Rel(repoPath, absTarget) - if err != nil { - return fmt.Errorf("failed to get relative path for symlink target %s: %w", absTarget, err) - } - - if strings.HasPrefix(relPath, "..") { - return fmt.Errorf("symlink target %s escapes the repository root %s", absTarget, repoPath) - } - // Recursively walk the symlink target _, err = DecryptFilesInDirectory(repoPath, absTarget) @@ -141,7 +124,7 @@ func DecryptFilesInDirectory(repoPath, dirPath string) ([]string, error) { return nil } - decrypted, err := DecryptFileInPlace(path, repoPath) + decrypted, err := DecryptFileInPlace(path) if err != nil { return fmt.Errorf("failed to decrypt file %s: %w", path, err) } @@ -174,7 +157,7 @@ func IsEncryptedContent(content string) bool { // DecryptFileInPlace decrypts a SOPS-encrypted file at the given path and overwrites it with the decrypted content. // If the file is encrypted and successfully decrypted, it returns true. If the file is not encrypted, it returns false without modifying the file. // The repoPath parameter is used to ensure that the file being decrypted is within the trusted repository root, preventing potential security issues with symlinks or path traversal. -func DecryptFileInPlace(path, repoPath string) (bool, error) { +func DecryptFileInPlace(path string) (bool, error) { path = filepath.Clean(path) if !filepath.IsAbs(path) { @@ -186,10 +169,6 @@ func DecryptFileInPlace(path, repoPath string) (bool, error) { return false, nil } - if repoPath == "" { - return false, fmt.Errorf("%w: trusted root must not be empty", filesystem.ErrInvalidFilePath) - } - lock := acquireFileLock(path) defer releaseFileLock(path, lock) @@ -202,38 +181,14 @@ func DecryptFileInPlace(path, repoPath string) (bool, error) { return false, nil } - // Prevent path to escape the repoPath - relPath, err := filepath.Rel(repoPath, path) - if err != nil { - return false, fmt.Errorf("failed to get relative path for symlink target %s: %w", path, err) - } - - if strings.HasPrefix(relPath, "..") { - return false, fmt.Errorf("path %s escapes the repository root %s", path, repoPath) - } - - // Ensure the path is within the trusted root and use the sanitized absolute path. - // Open the repository root for writing decrypted files without changing their permissions. - root, err := os.OpenRoot(repoPath) - if err != nil { - return false, fmt.Errorf("failed to open repo root %s: %w", repoPath, err) - } - defer root.Close() - decryptedContent, err := DecryptFile(path) if err != nil { return false, fmt.Errorf("failed to decrypt file %s: %w", path, err) } - f, err := root.OpenFile(relPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, filesystem.PermOwner) + err = os.WriteFile(path, decryptedContent, filesystem.PermOwner) if err != nil { - return false, fmt.Errorf("failed to open file %s for writing: %w", path, err) - } - - defer f.Close() - - if _, err := f.Write(decryptedContent); err != nil { - return false, fmt.Errorf("failed to write decrypted content to file %s: %w", path, err) + return false, fmt.Errorf("failed to write file %s: %w", path, err) } return true, nil diff --git a/doco-cd-src/internal/git/git.go b/doco-cd-src/internal/git/git.go index 2bb1440..35c8838 100644 --- a/doco-cd-src/internal/git/git.go +++ b/doco-cd-src/internal/git/git.go @@ -46,6 +46,7 @@ var ( ErrCheckoutFailed = errors.New("failed to checkout repository") ErrFetchFailed = errors.New("failed to fetch repository") ErrPullFailed = errors.New("failed to pull repository") + ErrRepositoryNotExists = git.ErrRepositoryNotExists ErrRepositoryAlreadyExists = git.ErrRepositoryAlreadyExists ErrInvalidReference = git.ErrInvalidReference ErrSSHKeyRequired = errors.New("ssh URL requires SSH_PRIVATE_KEY to be set") @@ -231,8 +232,8 @@ func OpenRepository(path string) (*git.Repository, error) { return git.PlainOpen(path) } -// fetchRepository fetches updates from the remote repository, including all branches and tags, and prunes deleted references. -func fetchRepository(repo *git.Repository, url string, skipTLSVerify bool, proxyOpts transport.ProxyOptions, auth transport.AuthMethod) error { +// FetchRepository fetches updates from the remote repository, including all branches and tags, and prunes deleted references. +func FetchRepository(repo *git.Repository, url string, skipTLSVerify bool, proxyOpts transport.ProxyOptions, auth transport.AuthMethod) error { opts := &git.FetchOptions{ RemoteName: RemoteName, RemoteURL: url, @@ -286,7 +287,7 @@ func UpdateRepository(path, url, ref string, skipTLSVerify bool, proxyOpts trans return nil, err } - err = fetchRepository(repo, url, skipTLSVerify, proxyOpts, auth) + err = FetchRepository(repo, url, skipTLSVerify, proxyOpts, auth) if err != nil { return nil, fmt.Errorf("%w: %w", ErrFetchFailed, err) } diff --git a/doco-cd-src/internal/stages/stage_1_init.go b/doco-cd-src/internal/stages/stage_1_init.go index 7d181c5..e5fca50 100644 --- a/doco-cd-src/internal/stages/stage_1_init.go +++ b/doco-cd-src/internal/stages/stage_1_init.go @@ -46,6 +46,8 @@ func (s *StageManager) RunInitStage(ctx context.Context, stageLog *slog.Logger) s.Repository.CloneURL = s.DeployConfig.RepositoryUrl s.Repository.Name = git.GetRepoName(string(s.Repository.CloneURL)) + // Load local (without remote: prefix) dotenv files before paths get updated to remote repository + // Remote dotenv files get read later err = config.LoadLocalDotEnv(s.DeployConfig, s.Repository.PathInternal) if err != nil { return fmt.Errorf("failed to parse local env files: %w", err) @@ -73,6 +75,22 @@ func (s *StageManager) RunInitStage(ctx context.Context, stageLog *slog.Logger) return fmt.Errorf("failed to get auth method: %w", err) } + // Attempt to fetch the remote repository before checking if we can skip cloning/updating, + // to ensure we have the latest commits and references available locally + if s.DeployConfig.RepositoryUrl != "" { + repo, err := git.OpenRepository(s.Repository.PathInternal) + switch { + case err == nil: + err = git.FetchRepository(repo, string(s.Repository.CloneURL), s.AppConfig.SkipTLSVerification, s.AppConfig.HttpProxy, auth) + if err != nil { + return fmt.Errorf("failed to fetch repository: %w", err) + } + case errors.Is(err, git.ErrRepositoryNotExists): // Continue without fetching the repository, it will be cloned later + default: + return fmt.Errorf("failed to open repository: %w", err) + } + } + // Check if we can skip cloning/updating because the previous run (initial or a prior deploy config) skipCloneUpdate, err := git.MatchesHead(s.Repository.PathInternal, s.DeployConfig.Reference) if err != nil { @@ -97,6 +115,12 @@ func (s *StageManager) RunInitStage(ctx context.Context, stageLog *slog.Logger) slog.String("url", string(s.Repository.CloneURL)), slog.String("path", s.Repository.PathExternal)) } + + // Now also load remote dotenv files + err = config.LoadLocalDotEnv(s.DeployConfig, filepath.Join(s.Repository.PathInternal, s.DeployConfig.WorkingDirectory)) + if err != nil { + return fmt.Errorf("failed to parse remote env files: %w", err) + } } if s.DeployConfig.Destroy { diff --git a/doco-cd-src/internal/stages/stage_2_pre-deploy.go b/doco-cd-src/internal/stages/stage_2_pre-deploy.go index f0a2533..3305980 100644 --- a/doco-cd-src/internal/stages/stage_2_pre-deploy.go +++ b/doco-cd-src/internal/stages/stage_2_pre-deploy.go @@ -5,16 +5,12 @@ import ( "errors" "fmt" "log/slog" - "os" "path/filepath" "strings" "time" "github.com/go-git/go-git/v5/plumbing" - "github.com/kimdre/doco-cd/internal/config" - "github.com/kimdre/doco-cd/internal/logger" - "github.com/kimdre/doco-cd/internal/utils/set" "github.com/kimdre/doco-cd/internal/docker" @@ -55,10 +51,18 @@ func (s *StageManager) RunPreDeployStage(ctx context.Context, stageLog *slog.Log stageLog.Debug("resolving external secrets", slog.Any("external_secrets", s.DeployConfig.ExternalSecrets)) // Resolve external secrets - s.DeployState.ResolvedSecrets, err = (*s.SecretProvider).ResolveSecretReferences(ctx, s.DeployConfig.ExternalSecrets) + resolvedSecrets, err := (*s.SecretProvider).ResolveSecretReferences(ctx, s.DeployConfig.ExternalSecrets) if err != nil { return fmt.Errorf("failed to resolve external secrets: %w", err) } + + if s.DeployConfig.Internal.Environment == nil { + s.DeployConfig.Internal.Environment = make(map[string]string) + } + + for k, v := range resolvedSecrets { + s.DeployConfig.Internal.Environment[k] = v + } } s.DeployConfig.Internal.Hash, err = s.DeployConfig.Hash() @@ -140,27 +144,10 @@ func (s *StageManager) RunPreDeployStage(ctx context.Context, stageLog *slog.Log return fmt.Errorf("failed to check for default compose files: %w", err) } - // Create a temporary env file if environment variables are specified in the deployment config - if s.DeployConfig.Internal.Environment != nil { - tmpEnvFile, err := config.CreateTmpDotEnvFile(s.DeployConfig) - if err != nil { - errMsg := "failed to create temporary env file" - return fmt.Errorf("%s: %w", errMsg, err) - } - - // Delete the temp file after deployment - defer func(name string) { - err = os.Remove(name) - if err != nil { - s.Log.Warn("failed to delete temporary env file", logger.ErrAttr(err), slog.String("file", name)) - } - }(tmpEnvFile) - } - s.Docker.Project, err = docker.LoadCompose( ctx, s.Repository.PathExternal, extAbsWorkingDir, s.DeployConfig.Name, s.DeployConfig.ComposeFiles, s.DeployConfig.EnvFiles, - s.DeployConfig.Profiles, s.DeployState.ResolvedSecrets) + s.DeployConfig.Profiles, s.DeployConfig.Internal.Environment) if err != nil { return fmt.Errorf("failed to load compose project: %w", err) } diff --git a/doco-cd-src/internal/stages/stage_3_deploy.go b/doco-cd-src/internal/stages/stage_3_deploy.go index e160100..7be0c06 100644 --- a/doco-cd-src/internal/stages/stage_3_deploy.go +++ b/doco-cd-src/internal/stages/stage_3_deploy.go @@ -66,7 +66,7 @@ func (s *StageManager) RunDeployStage(ctx context.Context, stageLog *slog.Logger err = docker.DeployStack(stageLog, s.Repository.PathExternal, &ctx, &s.Docker.Cmd, s.Docker.Client, s.Payload, s.DeployConfig, s.DeployState.ChangedFiles, latestCommit, config.AppVersion, - forceDeploy, s.DeployState.ResolvedSecrets) + forceDeploy) if err != nil { return fmt.Errorf("failed to deploy stack %s: %w", s.DeployConfig.Name, err) } diff --git a/doco-cd-src/internal/stages/types.go b/doco-cd-src/internal/stages/types.go index 979673d..6adcdcc 100644 --- a/doco-cd-src/internal/stages/types.go +++ b/doco-cd-src/internal/stages/types.go @@ -16,7 +16,6 @@ import ( "github.com/kimdre/doco-cd/internal/config" gitInternal "github.com/kimdre/doco-cd/internal/git" "github.com/kimdre/doco-cd/internal/secretprovider" - secrettypes "github.com/kimdre/doco-cd/internal/secretprovider/types" "github.com/kimdre/doco-cd/internal/webhook" ) @@ -124,8 +123,7 @@ type Docker struct { // DeploymentState holds the dynamic state information during the deployment process. type DeploymentState struct { - ChangedFiles []gitInternal.ChangedFile - ResolvedSecrets secrettypes.ResolvedSecrets + ChangedFiles []gitInternal.ChangedFile } // StageManager is the main structure that holds the logger and stage data. diff --git a/doco-cd-src/test/test.compose.yaml b/doco-cd-src/test/test.compose.yaml index 64bb217..502671d 100644 --- a/doco-cd-src/test/test.compose.yaml +++ b/doco-cd-src/test/test.compose.yaml @@ -14,6 +14,8 @@ services: labels: - something=${SOMETHING:-none} - base=${BASE:-none} + - prod=${PROD:-none} + - remote=${REMOTE:-none} env_file: - test.env secrets: