diff --git a/cmd/worker.go b/cmd/worker.go index c4f630d60..43f74ef7b 100644 --- a/cmd/worker.go +++ b/cmd/worker.go @@ -76,6 +76,7 @@ func (t *TemporalWorker) NewServer(userConfig legacy.UserConfig, config legacy.C Metrics: globalCfg.Metrics, LyftAuditJobsSnsTopicArn: userConfig.LyftAuditJobsSnsTopicArn, RevisionSetter: globalCfg.RevisionSetter, + DeploymentSweeper: globalCfg.DeploymentSweeper, } return temporalworker.NewServer(cfg) } diff --git a/server/config/raw/deployment_sweeper.go b/server/config/raw/deployment_sweeper.go new file mode 100644 index 000000000..396415af0 --- /dev/null +++ b/server/config/raw/deployment_sweeper.go @@ -0,0 +1,36 @@ +package raw + +import ( + "time" + + "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/config/valid" +) + +// DeploymentSweeper configures the temporal worker's cleanup of orphaned deployment directories. +// Leaving it out is safe: the sweeper then only logs what it would delete. +type DeploymentSweeper struct { + // MaxAge is a Go duration such as "360h". Empty uses the default of 15 days. + MaxAge string `yaml:"max_age" json:"max_age"` + // Delete turns on actual deletion. When false the sweeper only logs. + Delete bool `yaml:"delete" json:"delete"` +} + +func (d DeploymentSweeper) Validate() error { + if d.MaxAge == "" { + return nil + } + if _, err := time.ParseDuration(d.MaxAge); err != nil { + return errors.Wrap(err, "max_age must be a duration such as 360h") + } + return nil +} + +func (d DeploymentSweeper) ToValid() valid.DeploymentSweeper { + // Validate has already checked the format, an empty value stays zero and gets the default. + maxAge, _ := time.ParseDuration(d.MaxAge) + return valid.DeploymentSweeper{ + MaxAge: maxAge, + Delete: d.Delete, + } +} diff --git a/server/config/raw/deployment_sweeper_test.go b/server/config/raw/deployment_sweeper_test.go new file mode 100644 index 000000000..712cf41f1 --- /dev/null +++ b/server/config/raw/deployment_sweeper_test.go @@ -0,0 +1,53 @@ +package raw_test + +import ( + "testing" + "time" + + "github.com/runatlantis/atlantis/server/config/raw" + "github.com/runatlantis/atlantis/server/config/valid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" +) + +func TestDeploymentSweeper_ToValid(t *testing.T) { + cases := []struct { + description string + input string + expected valid.DeploymentSweeper + }{ + { + description: "left out is a dry run with the default max age", + input: "{}", + expected: valid.DeploymentSweeper{}, + }, + { + description: "delete with a max age", + input: "max_age: 360h\ndelete: true", + expected: valid.DeploymentSweeper{MaxAge: 360 * time.Hour, Delete: true}, + }, + { + description: "max age only stays a dry run", + input: "max_age: 48h", + expected: valid.DeploymentSweeper{MaxAge: 48 * time.Hour}, + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + var cfg raw.DeploymentSweeper + require.NoError(t, yaml.Unmarshal([]byte(c.input), &cfg)) + require.NoError(t, cfg.Validate()) + assert.Equal(t, c.expected, cfg.ToValid()) + }) + } +} + +func TestDeploymentSweeper_ValidateRejectsBadMaxAge(t *testing.T) { + for _, maxAge := range []string{"15d", "soon", "-"} { + t.Run(maxAge, func(t *testing.T) { + assert.Error(t, raw.DeploymentSweeper{MaxAge: maxAge}.Validate()) + }) + } +} diff --git a/server/config/raw/global_cfg.go b/server/config/raw/global_cfg.go index 90a9476b0..4d437b0e7 100644 --- a/server/config/raw/global_cfg.go +++ b/server/config/raw/global_cfg.go @@ -25,6 +25,7 @@ type GlobalCfg struct { Github Github `yaml:"github" json:"github"` Persistence Persistence `yaml:"persistence" json:"persistence"` RevisionSetter RevisionSetter `yaml:"revision_setter" json:"revision_setter"` + DeploymentSweeper DeploymentSweeper `yaml:"deployment_sweeper" json:"deployment_sweeper"` Admin Admin `yaml:"admin" json:"admin"` AdhocMode AdhocMode `yaml:"adhoc_mode" json:"adhoc_mode"` } @@ -132,6 +133,7 @@ func (g GlobalCfg) Validate() error { validation.Field(&g.Github), validation.Field(&g.TerraformLogFilters), validation.Field(&g.Persistence), + validation.Field(&g.DeploymentSweeper), ) if err != nil { return err @@ -214,6 +216,7 @@ func (g GlobalCfg) ToValid(defaultCfg valid.GlobalCfg) valid.GlobalCfg { Github: g.Github.ToValid(), Admin: g.Admin.ToValid(), RevisionSetter: g.RevisionSetter.ToValid(), + DeploymentSweeper: g.DeploymentSweeper.ToValid(), AdhocMode: g.AdhocMode.ToValid(), } } diff --git a/server/config/valid/global_cfg.go b/server/config/valid/global_cfg.go index f76d7cafd..51ba96dbb 100644 --- a/server/config/valid/global_cfg.go +++ b/server/config/valid/global_cfg.go @@ -2,6 +2,7 @@ package valid import ( "regexp" + "time" "github.com/graymeta/stow" "github.com/graymeta/stow/local" @@ -62,6 +63,7 @@ type GlobalCfg struct { Temporal Temporal Github Github RevisionSetter RevisionSetter + DeploymentSweeper DeploymentSweeper Admin Admin AdhocMode AdhocMode } @@ -168,6 +170,13 @@ type TaskQueue struct { ActivitiesPerSecond float64 } +// DeploymentSweeper configures cleanup of orphaned deployment directories. The zero value means a dry run +// with the default max age. +type DeploymentSweeper struct { + MaxAge time.Duration + Delete bool +} + type RevisionSetter struct { BasicAuth BasicAuth URL string diff --git a/server/neptune/sync/crons/deployment_sweeper.go b/server/neptune/sync/crons/deployment_sweeper.go new file mode 100644 index 000000000..bdb2c1d8d --- /dev/null +++ b/server/neptune/sync/crons/deployment_sweeper.go @@ -0,0 +1,152 @@ +package crons + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally/v4" +) + +const ( + // deploymentsDirName must match the directory GithubFetchRoot creates deployment directories in. + deploymentsDirName = "deployments" + + // DefaultDeploymentMaxAge is how old a deployment directory must be before it is considered orphaned. + // Terraform runs finish in minutes (an apply can take up to an hour), so a directory this old can't be in use. + DefaultDeploymentMaxAge = 15 * 24 * time.Hour + + // MinDeploymentMaxAge guards against a misconfiguration deleting the working directories of runs in flight. + MinDeploymentMaxAge = 24 * time.Hour +) + +// DeploymentSweeper deletes deployment directories that were left behind on this worker's volume. +// +// Each Terraform run extracts the repo into /deployments/ and deletes it when the run +// finishes. A run that never reaches its cleanup (a failed or interrupted fetch, a killed pod) leaves the +// directory behind and nothing else removes it, so the volume slowly fills up. The sweeper is the backstop +// for those directories. It only looks at the immediate children of /deployments, so plugin caches +// and everything else under DataDir are never touched. +type DeploymentSweeper struct { + dir string + maxAge time.Duration + delete bool + logger logging.Logger + metrics deploymentSweeperMetrics + now func() time.Time +} + +type deploymentSweeperMetrics struct { + scanned tally.Gauge + stale tally.Counter + deleted tally.Counter + errors tally.Counter +} + +// NewDeploymentSweeper returns a sweeper for the deployments directory under dataDir. A zero maxAge uses +// DefaultDeploymentMaxAge. Unless deleteStale is true the sweeper only logs what it would delete. +func NewDeploymentSweeper(dataDir string, maxAge time.Duration, deleteStale bool, logger logging.Logger, scope tally.Scope) (*DeploymentSweeper, error) { + if dataDir == "" || filepath.Clean(dataDir) == string(filepath.Separator) { + return nil, errors.Errorf("refusing to sweep deployments under data dir %q", dataDir) + } + if maxAge == 0 { + maxAge = DefaultDeploymentMaxAge + } + if maxAge < MinDeploymentMaxAge { + return nil, errors.Errorf("deployment sweeper max age %s is below the minimum of %s", maxAge, MinDeploymentMaxAge) + } + + scope = scope.SubScope("deployment_sweeper") + return &DeploymentSweeper{ + dir: filepath.Join(dataDir, deploymentsDirName), + maxAge: maxAge, + delete: deleteStale, + logger: logger, + metrics: deploymentSweeperMetrics{ + scanned: scope.Gauge("scanned"), + stale: scope.Counter("stale"), + deleted: scope.Counter("deleted"), + errors: scope.Counter("errors"), + }, + now: time.Now, + }, nil +} + +// Run does one sweep. Failing to delete one directory does not stop the sweep. +func (s *DeploymentSweeper) Run(ctx context.Context) error { + entries, err := os.ReadDir(s.dir) + if os.IsNotExist(err) { + // nothing has been fetched on this worker yet + return nil + } + if err != nil { + s.metrics.errors.Inc(1) + return errors.Wrapf(err, "reading %s", s.dir) + } + + var scanned, stale, deleted, failed int + for _, entry := range entries { + if ctx.Err() != nil { + s.logger.WarnContext(ctx, "deployment sweep interrupted") + break + } + + // only real directories: symlinks and stray files are left alone + if !entry.Type().IsDir() { + continue + } + scanned++ + + info, err := entry.Info() + if err != nil { + // the directory was removed while we were scanning + if !os.IsNotExist(err) { + failed++ + s.logger.WarnContext(ctx, "unable to stat deployment directory", map[string]interface{}{ + "path": filepath.Join(s.dir, entry.Name()), + "error": err.Error(), + }) + } + continue + } + + age := s.now().Sub(info.ModTime()) + if age <= s.maxAge { + continue + } + stale++ + + path := filepath.Join(s.dir, entry.Name()) + fields := map[string]interface{}{ + "path": path, + "age": age.Round(time.Hour).String(), + "dry_run": !s.delete, + } + if !s.delete { + s.logger.InfoContext(ctx, "stale deployment directory found", fields) + continue + } + + if err := os.RemoveAll(path); err != nil { + failed++ + fields["error"] = err.Error() + s.logger.ErrorContext(ctx, "unable to delete stale deployment directory", fields) + continue + } + deleted++ + s.logger.InfoContext(ctx, "deleted stale deployment directory", fields) + } + + s.metrics.scanned.Update(float64(scanned)) + s.metrics.stale.Inc(int64(stale)) + s.metrics.deleted.Inc(int64(deleted)) + s.metrics.errors.Inc(int64(failed)) + + s.logger.InfoContext(ctx, fmt.Sprintf("deployment sweep finished: %d directories, %d stale, %d deleted, %d errors (dry run: %t)", + scanned, stale, deleted, failed, !s.delete)) + return nil +} diff --git a/server/neptune/sync/crons/deployment_sweeper_test.go b/server/neptune/sync/crons/deployment_sweeper_test.go new file mode 100644 index 000000000..14aba5e31 --- /dev/null +++ b/server/neptune/sync/crons/deployment_sweeper_test.go @@ -0,0 +1,185 @@ +package crons_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/neptune/sync/crons" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/uber-go/tally/v4" +) + +const day = 24 * time.Hour + +// sweeperDataDir builds a data dir like a worker's: deployments with one directory per run, plus siblings +// the sweeper must never touch. +type sweeperDataDir struct { + root string + t *testing.T +} + +func newSweeperDataDir(t *testing.T) sweeperDataDir { + t.Helper() + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "deployments"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(root, "plugin-cache", "provider"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "plugin-cache", "provider", "binary"), []byte("x"), 0o600)) + return sweeperDataDir{root: root, t: t} +} + +func (d sweeperDataDir) deployment(name string, age time.Duration) string { + d.t.Helper() + path := filepath.Join(d.root, "deployments", name) + require.NoError(d.t, os.MkdirAll(filepath.Join(path, "repo"), 0o755)) + require.NoError(d.t, os.WriteFile(filepath.Join(path, "repo", "main.tf"), []byte("x"), 0o600)) + old := time.Now().Add(-age) + require.NoError(d.t, os.Chtimes(path, old, old)) + return path +} + +func (d sweeperDataDir) exists(path string) bool { + _, err := os.Lstat(path) + return err == nil +} + +func newSweeper(t *testing.T, dataDir string, maxAge time.Duration, deleteStale bool) *crons.DeploymentSweeper { + t.Helper() + sweeper, err := crons.NewDeploymentSweeper(dataDir, maxAge, deleteStale, logging.NewNoopCtxLogger(t), tally.NoopScope) + require.NoError(t, err) + return sweeper +} + +func TestDeploymentSweeper_DeletesOnlyStaleDirectories(t *testing.T) { + d := newSweeperDataDir(t) + stale := d.deployment("stale", 20*day) + older := d.deployment("older", 200*day) + recent := d.deployment("recent", 2*day) + inFlight := d.deployment("in-flight", time.Minute) + + require.NoError(t, newSweeper(t, d.root, 15*day, true).Run(context.Background())) + + assert.False(t, d.exists(stale)) + assert.False(t, d.exists(older)) + assert.True(t, d.exists(recent)) + assert.True(t, d.exists(inFlight)) +} + +func TestDeploymentSweeper_UsesDefaultMaxAge(t *testing.T) { + d := newSweeperDataDir(t) + stale := d.deployment("stale", 16*day) + recent := d.deployment("recent", 14*day) + + require.NoError(t, newSweeper(t, d.root, 0, true).Run(context.Background())) + + assert.False(t, d.exists(stale)) + assert.True(t, d.exists(recent)) +} + +func TestDeploymentSweeper_DryRunDeletesNothing(t *testing.T) { + d := newSweeperDataDir(t) + stale := d.deployment("stale", 20*day) + recent := d.deployment("recent", 2*day) + + require.NoError(t, newSweeper(t, d.root, 15*day, false).Run(context.Background())) + + assert.True(t, d.exists(stale)) + assert.True(t, d.exists(recent)) +} + +func TestDeploymentSweeper_LeavesEverythingElseAlone(t *testing.T) { + d := newSweeperDataDir(t) + d.deployment("stale", 20*day) + + // a stray file and a symlink in the deployments dir, both old + stray := filepath.Join(d.root, "deployments", "stray-file") + require.NoError(t, os.WriteFile(stray, []byte("x"), 0o600)) + // a symlink to a directory outside deployments must be left alone, and so must its target + target := filepath.Join(d.root, "plugin-cache") + link := filepath.Join(d.root, "deployments", "link") + require.NoError(t, os.Symlink(target, link)) + old := time.Now().Add(-100 * day) + require.NoError(t, os.Chtimes(stray, old, old)) + + require.NoError(t, newSweeper(t, d.root, 15*day, true).Run(context.Background())) + + assert.True(t, d.exists(stray)) + assert.True(t, d.exists(link)) + assert.True(t, d.exists(filepath.Join(target, "provider", "binary"))) + assert.True(t, d.exists(filepath.Join(d.root, "deployments"))) +} + +func TestDeploymentSweeper_OldDeploymentsDirItselfIsNotDeleted(t *testing.T) { + d := newSweeperDataDir(t) + deployments := filepath.Join(d.root, "deployments") + old := time.Now().Add(-365 * day) + require.NoError(t, os.Chtimes(deployments, old, old)) + + require.NoError(t, newSweeper(t, d.root, 15*day, true).Run(context.Background())) + + assert.True(t, d.exists(deployments)) +} + +func TestDeploymentSweeper_MissingDeploymentsDir(t *testing.T) { + root := t.TempDir() + + assert.NoError(t, newSweeper(t, root, 15*day, true).Run(context.Background())) +} + +func TestDeploymentSweeper_StopsWhenContextCancelled(t *testing.T) { + d := newSweeperDataDir(t) + stale := d.deployment("stale", 20*day) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + require.NoError(t, newSweeper(t, d.root, 15*day, true).Run(ctx)) + + assert.True(t, d.exists(stale)) +} + +func TestDeploymentSweeper_ContinuesAfterDeleteFailure(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permissions are not enforced for root") + } + d := newSweeperDataDir(t) + // a directory that can't be emptied because its child directory is read-only + stuck := d.deployment("a-stuck", 20*day) + locked := filepath.Join(stuck, "repo") + require.NoError(t, os.Chmod(locked, 0o500)) + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + other := d.deployment("z-other", 20*day) + + require.NoError(t, newSweeper(t, d.root, 15*day, true).Run(context.Background())) + + assert.True(t, d.exists(stuck)) + assert.False(t, d.exists(other)) +} + +func TestNewDeploymentSweeper_RejectsUnsafeConfig(t *testing.T) { + cases := []struct { + description string + dataDir string + maxAge time.Duration + }{ + {description: "empty data dir", dataDir: "", maxAge: 15 * day}, + {description: "root data dir", dataDir: "/", maxAge: 15 * day}, + {description: "max age below the minimum", dataDir: t.TempDir(), maxAge: time.Hour}, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + _, err := crons.NewDeploymentSweeper(c.dataDir, c.maxAge, true, logging.NewNoopCtxLogger(t), tally.NoopScope) + assert.Error(t, err) + }) + } +} + +func TestNewDeploymentSweeper_AcceptsMinimumMaxAge(t *testing.T) { + _, err := crons.NewDeploymentSweeper(t.TempDir(), crons.MinDeploymentMaxAge, true, logging.NewNoopCtxLogger(t), tally.NoopScope) + assert.NoError(t, err) +} diff --git a/server/neptune/temporalworker/config/config.go b/server/neptune/temporalworker/config/config.go index 1cccf2a4f..634a1296a 100644 --- a/server/neptune/temporalworker/config/config.go +++ b/server/neptune/temporalworker/config/config.go @@ -59,4 +59,5 @@ type Config struct { CtxLogger logging.Logger App githubapp.Config LyftAuditJobsSnsTopicArn string + DeploymentSweeper valid.DeploymentSweeper } diff --git a/server/neptune/temporalworker/server.go b/server/neptune/temporalworker/server.go index 62d9a9f9a..ab6a1c6e4 100644 --- a/server/neptune/temporalworker/server.go +++ b/server/neptune/temporalworker/server.go @@ -64,6 +64,9 @@ const ( // 5 minutes to allow cleaning up the job store StreamHandlerTimeout = 5 * time.Minute PRRevisionTaskQueueActivitiesPerSecond = 2 + + // how often each worker looks for orphaned deployment directories + deploymentSweepFrequency = 6 * time.Hour ) type Server struct { @@ -217,6 +220,17 @@ func NewServer(config *config.Config) (*Server, error) { InstallationID: config.GithubCfg.TemporalAppInstallationID, } + deploymentSweeper, err := crons.NewDeploymentSweeper( + config.DataDir, + config.DeploymentSweeper.MaxAge, + config.DeploymentSweeper.Delete, + config.CtxLogger, + scope, + ) + if err != nil { + return nil, errors.Wrap(err, "initializing deployment sweeper") + } + cronScheduler := internalSync.NewCronScheduler(config.CtxLogger) server := Server{ @@ -231,6 +245,13 @@ func NewServer(config *config.Config) (*Server, error) { Executor: crons.NewRateLimitStats(scope, clientCreator, config.GithubCfg.TemporalAppInstallationID).Run, Frequency: 1 * time.Minute, }, + { + // Each worker pod has its own volume, so every pod sweeps its own deployments directory. Runs + // every few hours rather than daily because the first run only happens a full interval after + // startup, and pods restart on every deploy. + Executor: deploymentSweeper.Run, + Frequency: deploymentSweepFrequency, + }, }, HTTPServerProxy: httpServerProxy, Port: config.ServerCfg.Port,