Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func (t *TemporalWorker) NewServer(userConfig legacy.UserConfig, config legacy.C
Metrics: globalCfg.Metrics,
LyftAuditJobsSnsTopicArn: userConfig.LyftAuditJobsSnsTopicArn,
RevisionSetter: globalCfg.RevisionSetter,
DeploymentSweeper: globalCfg.DeploymentSweeper,
}
return temporalworker.NewServer(cfg)
}
36 changes: 36 additions & 0 deletions server/config/raw/deployment_sweeper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package raw

import (
"time"

"github.com/pkg/errors"
"github.com/runatlantis/atlantis/server/config/valid"
)

// DeploymentSweeper configures the temporal worker's cleanup of orphaned deployment directories.
// Leaving it out is safe: the sweeper then only logs what it would delete.
type DeploymentSweeper struct {
// MaxAge is a Go duration such as "360h". Empty uses the default of 15 days.
MaxAge string `yaml:"max_age" json:"max_age"`
// Delete turns on actual deletion. When false the sweeper only logs.
Delete bool `yaml:"delete" json:"delete"`
}

func (d DeploymentSweeper) Validate() error {
if d.MaxAge == "" {
return nil
}
if _, err := time.ParseDuration(d.MaxAge); err != nil {
return errors.Wrap(err, "max_age must be a duration such as 360h")
}
return nil
}

func (d DeploymentSweeper) ToValid() valid.DeploymentSweeper {
// Validate has already checked the format, an empty value stays zero and gets the default.
maxAge, _ := time.ParseDuration(d.MaxAge)
return valid.DeploymentSweeper{
MaxAge: maxAge,
Delete: d.Delete,
}
}
53 changes: 53 additions & 0 deletions server/config/raw/deployment_sweeper_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package raw_test

import (
"testing"
"time"

"github.com/runatlantis/atlantis/server/config/raw"
"github.com/runatlantis/atlantis/server/config/valid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v2"
)

func TestDeploymentSweeper_ToValid(t *testing.T) {
cases := []struct {
description string
input string
expected valid.DeploymentSweeper
}{
{
description: "left out is a dry run with the default max age",
input: "{}",
expected: valid.DeploymentSweeper{},
},
{
description: "delete with a max age",
input: "max_age: 360h\ndelete: true",
expected: valid.DeploymentSweeper{MaxAge: 360 * time.Hour, Delete: true},
},
{
description: "max age only stays a dry run",
input: "max_age: 48h",
expected: valid.DeploymentSweeper{MaxAge: 48 * time.Hour},
},
}

for _, c := range cases {
t.Run(c.description, func(t *testing.T) {
var cfg raw.DeploymentSweeper
require.NoError(t, yaml.Unmarshal([]byte(c.input), &cfg))
require.NoError(t, cfg.Validate())
assert.Equal(t, c.expected, cfg.ToValid())
})
}
}

func TestDeploymentSweeper_ValidateRejectsBadMaxAge(t *testing.T) {
for _, maxAge := range []string{"15d", "soon", "-"} {
t.Run(maxAge, func(t *testing.T) {
assert.Error(t, raw.DeploymentSweeper{MaxAge: maxAge}.Validate())
})
}
}
3 changes: 3 additions & 0 deletions server/config/raw/global_cfg.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type GlobalCfg struct {
Github Github `yaml:"github" json:"github"`
Persistence Persistence `yaml:"persistence" json:"persistence"`
RevisionSetter RevisionSetter `yaml:"revision_setter" json:"revision_setter"`
DeploymentSweeper DeploymentSweeper `yaml:"deployment_sweeper" json:"deployment_sweeper"`
Admin Admin `yaml:"admin" json:"admin"`
AdhocMode AdhocMode `yaml:"adhoc_mode" json:"adhoc_mode"`
}
Expand Down Expand Up @@ -132,6 +133,7 @@ func (g GlobalCfg) Validate() error {
validation.Field(&g.Github),
validation.Field(&g.TerraformLogFilters),
validation.Field(&g.Persistence),
validation.Field(&g.DeploymentSweeper),
)
if err != nil {
return err
Expand Down Expand Up @@ -214,6 +216,7 @@ func (g GlobalCfg) ToValid(defaultCfg valid.GlobalCfg) valid.GlobalCfg {
Github: g.Github.ToValid(),
Admin: g.Admin.ToValid(),
RevisionSetter: g.RevisionSetter.ToValid(),
DeploymentSweeper: g.DeploymentSweeper.ToValid(),
AdhocMode: g.AdhocMode.ToValid(),
}
}
Expand Down
9 changes: 9 additions & 0 deletions server/config/valid/global_cfg.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package valid

import (
"regexp"
"time"

"github.com/graymeta/stow"
"github.com/graymeta/stow/local"
Expand Down Expand Up @@ -62,6 +63,7 @@ type GlobalCfg struct {
Temporal Temporal
Github Github
RevisionSetter RevisionSetter
DeploymentSweeper DeploymentSweeper
Admin Admin
AdhocMode AdhocMode
}
Expand Down Expand Up @@ -168,6 +170,13 @@ type TaskQueue struct {
ActivitiesPerSecond float64
}

// DeploymentSweeper configures cleanup of orphaned deployment directories. The zero value means a dry run
// with the default max age.
type DeploymentSweeper struct {
MaxAge time.Duration
Delete bool
}

type RevisionSetter struct {
BasicAuth BasicAuth
URL string
Expand Down
152 changes: 152 additions & 0 deletions server/neptune/sync/crons/deployment_sweeper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
package crons

import (
"context"
"fmt"
"os"
"path/filepath"
"time"

"github.com/pkg/errors"
"github.com/runatlantis/atlantis/server/logging"
"github.com/uber-go/tally/v4"
)

const (
// deploymentsDirName must match the directory GithubFetchRoot creates deployment directories in.
deploymentsDirName = "deployments"

// DefaultDeploymentMaxAge is how old a deployment directory must be before it is considered orphaned.
// Terraform runs finish in minutes (an apply can take up to an hour), so a directory this old can't be in use.
DefaultDeploymentMaxAge = 15 * 24 * time.Hour

// MinDeploymentMaxAge guards against a misconfiguration deleting the working directories of runs in flight.
MinDeploymentMaxAge = 24 * time.Hour
)

// DeploymentSweeper deletes deployment directories that were left behind on this worker's volume.
//
// Each Terraform run extracts the repo into <DataDir>/deployments/<deployment id> and deletes it when the run
// finishes. A run that never reaches its cleanup (a failed or interrupted fetch, a killed pod) leaves the
// directory behind and nothing else removes it, so the volume slowly fills up. The sweeper is the backstop
// for those directories. It only looks at the immediate children of <DataDir>/deployments, so plugin caches
// and everything else under DataDir are never touched.
type DeploymentSweeper struct {
dir string
maxAge time.Duration
delete bool
logger logging.Logger
metrics deploymentSweeperMetrics
now func() time.Time
}

type deploymentSweeperMetrics struct {
scanned tally.Gauge
stale tally.Counter
deleted tally.Counter
errors tally.Counter
}

// NewDeploymentSweeper returns a sweeper for the deployments directory under dataDir. A zero maxAge uses
// DefaultDeploymentMaxAge. Unless deleteStale is true the sweeper only logs what it would delete.
func NewDeploymentSweeper(dataDir string, maxAge time.Duration, deleteStale bool, logger logging.Logger, scope tally.Scope) (*DeploymentSweeper, error) {
if dataDir == "" || filepath.Clean(dataDir) == string(filepath.Separator) {
return nil, errors.Errorf("refusing to sweep deployments under data dir %q", dataDir)
}
if maxAge == 0 {
maxAge = DefaultDeploymentMaxAge
}
if maxAge < MinDeploymentMaxAge {
return nil, errors.Errorf("deployment sweeper max age %s is below the minimum of %s", maxAge, MinDeploymentMaxAge)
}

scope = scope.SubScope("deployment_sweeper")
return &DeploymentSweeper{
dir: filepath.Join(dataDir, deploymentsDirName),
maxAge: maxAge,
delete: deleteStale,
logger: logger,
metrics: deploymentSweeperMetrics{
scanned: scope.Gauge("scanned"),
stale: scope.Counter("stale"),
deleted: scope.Counter("deleted"),
errors: scope.Counter("errors"),
},
now: time.Now,
}, nil
}

// Run does one sweep. Failing to delete one directory does not stop the sweep.
func (s *DeploymentSweeper) Run(ctx context.Context) error {
entries, err := os.ReadDir(s.dir)
if os.IsNotExist(err) {
// nothing has been fetched on this worker yet
return nil
}
if err != nil {
s.metrics.errors.Inc(1)
return errors.Wrapf(err, "reading %s", s.dir)
}

var scanned, stale, deleted, failed int
for _, entry := range entries {
if ctx.Err() != nil {
s.logger.WarnContext(ctx, "deployment sweep interrupted")
break
}

// only real directories: symlinks and stray files are left alone
if !entry.Type().IsDir() {
continue
}
scanned++

info, err := entry.Info()
if err != nil {
// the directory was removed while we were scanning
if !os.IsNotExist(err) {
failed++
s.logger.WarnContext(ctx, "unable to stat deployment directory", map[string]interface{}{
"path": filepath.Join(s.dir, entry.Name()),
"error": err.Error(),
})
}
continue
}

age := s.now().Sub(info.ModTime())
if age <= s.maxAge {
continue
}
stale++

path := filepath.Join(s.dir, entry.Name())
fields := map[string]interface{}{
"path": path,
"age": age.Round(time.Hour).String(),
"dry_run": !s.delete,
}
if !s.delete {
s.logger.InfoContext(ctx, "stale deployment directory found", fields)
continue
}

if err := os.RemoveAll(path); err != nil {
failed++
fields["error"] = err.Error()
s.logger.ErrorContext(ctx, "unable to delete stale deployment directory", fields)
continue
}
deleted++
s.logger.InfoContext(ctx, "deleted stale deployment directory", fields)
}

s.metrics.scanned.Update(float64(scanned))
s.metrics.stale.Inc(int64(stale))
s.metrics.deleted.Inc(int64(deleted))
s.metrics.errors.Inc(int64(failed))

s.logger.InfoContext(ctx, fmt.Sprintf("deployment sweep finished: %d directories, %d stale, %d deleted, %d errors (dry run: %t)",
scanned, stale, deleted, failed, !s.delete))
return nil
}
Loading
Loading