diff --git a/server/neptune/temporalworker/job/receiver_registry_test.go b/server/neptune/temporalworker/job/receiver_registry_test.go index 0ebdcebc6..c1b71c1f5 100644 --- a/server/neptune/temporalworker/job/receiver_registry_test.go +++ b/server/neptune/temporalworker/job/receiver_registry_test.go @@ -89,7 +89,9 @@ func TestReceiverRegistry(t *testing.T) { t.Run("adds a receiver and broadcast", func(t *testing.T) { recvRegistry := job.NewReceiverRegistry() - ch := make(chan string) + // Broadcast doesn't wait for a receiver (it drops one that isn't ready), so with an unbuffered + // channel the test hangs whenever Broadcast runs before the read below. + ch := make(chan string, 1) recvRegistry.AddReceiver(jobID, ch) go recvRegistry.Broadcast(job.OutputLine{ diff --git a/server/neptune/temporalworker/job/stream_handler.go b/server/neptune/temporalworker/job/stream_handler.go index dbdb80d12..bd2e04911 100644 --- a/server/neptune/temporalworker/job/stream_handler.go +++ b/server/neptune/temporalworker/job/stream_handler.go @@ -5,6 +5,7 @@ import ( "fmt" "sync" + "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/config/valid" "github.com/runatlantis/atlantis/server/legacy/events/terraform/filter" "github.com/runatlantis/atlantis/server/logging" @@ -78,6 +79,35 @@ func (s *StreamHandler) CloseJob(ctx context.Context, jobID string) error { return s.Store.Close(ctx, jobID, Complete) } +// ApplyDivider is written between the copied plan output and the apply's own output. +const ApplyDivider = "---- terraform apply ----" + +// CopyJob appends the output recorded for fromJobID to toJobID, followed by ApplyDivider, ahead of anything +// toJobID writes afterwards. It is used to start an apply job's log with the plan it applies, so the apply log +// (which is what gets persisted and picked up downstream) shows plan and apply together. +// +// It is not idempotent: calling it again appends the lines again, so callers must not retry it. +func (s *StreamHandler) CopyJob(ctx context.Context, fromJobID string, toJobID string) error { + from, err := s.Store.Get(ctx, fromJobID) + if err != nil { + return errors.Wrapf(err, "getting job %s", fromJobID) + } + if from == nil || len(from.Output) == 0 { + return nil + } + + // copy the lines up front: the source is complete but may still be shared with the store + lines := append([]string(nil), from.Output...) + lines = append(lines, ApplyDivider) + for _, line := range lines { + s.ReceiverRegistry.Broadcast(OutputLine{JobID: toJobID, Line: line}) + if err := s.Store.Write(ctx, toJobID, line); err != nil { + return errors.Wrapf(err, "writing to job %s", toJobID) + } + } + return nil +} + func (s *StreamHandler) CleanUp(ctx context.Context) error { s.Logger.InfoContext(ctx, "waiting for jobs to finish") // Should we timeout here? We do risk a deadlock without this however adding a timeout also risks diff --git a/server/neptune/temporalworker/job/stream_handler_copy_test.go b/server/neptune/temporalworker/job/stream_handler_copy_test.go new file mode 100644 index 000000000..a915210eb --- /dev/null +++ b/server/neptune/temporalworker/job/stream_handler_copy_test.go @@ -0,0 +1,119 @@ +package job_test + +import ( + "context" + "errors" + "testing" + + "github.com/runatlantis/atlantis/server/config/valid" + "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/neptune/temporalworker/job" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type copyStore struct { + jobs map[string]*job.Job + getErr error + writeErr error + writeLog []string + writeKeys []string +} + +func (s *copyStore) Get(_ context.Context, jobID string) (*job.Job, error) { + if s.getErr != nil { + return nil, s.getErr + } + return s.jobs[jobID], nil +} + +func (s *copyStore) Write(_ context.Context, jobID string, output string) error { + if s.writeErr != nil { + return s.writeErr + } + s.writeKeys = append(s.writeKeys, jobID) + s.writeLog = append(s.writeLog, output) + return nil +} + +func (s *copyStore) Remove(string) {} +func (s *copyStore) Close(context.Context, string, job.JobStatus) error { return nil } +func (s *copyStore) Cleanup(context.Context) error { return nil } + +type copyRegistry struct { + broadcast []job.OutputLine +} + +func (r *copyRegistry) AddReceiver(string, chan string) {} +func (r *copyRegistry) Broadcast(msg job.OutputLine) { r.broadcast = append(r.broadcast, msg) } +func (r *copyRegistry) CleanUp() {} +func (r *copyRegistry) Close(context.Context, string) {} + +func newCopyHandler(t *testing.T, store *copyStore, registry *copyRegistry) *job.StreamHandler { + t.Helper() + return job.NewStreamHandler(store, registry, valid.TerraformLogFilters{}, logging.NewNoopCtxLogger(t)) +} + +func TestStreamHandler_CopyJob(t *testing.T) { + t.Run("writes the source output to the destination in order and broadcasts it", func(t *testing.T) { + store := ©Store{jobs: map[string]*job.Job{ + "plan": {Status: job.Complete, Output: []string{"plan 1", "plan 2"}}, + }} + registry := ©Registry{} + + require.NoError(t, newCopyHandler(t, store, registry).CopyJob(context.Background(), "plan", "apply")) + + assert.Equal(t, []string{"plan 1", "plan 2", job.ApplyDivider}, store.writeLog) + assert.Equal(t, []string{"apply", "apply", "apply"}, store.writeKeys) + assert.Equal(t, []job.OutputLine{ + {JobID: "apply", Line: "plan 1"}, + {JobID: "apply", Line: "plan 2"}, + {JobID: "apply", Line: job.ApplyDivider}, + }, registry.broadcast) + }) + + t.Run("later apply output follows the copied plan", func(t *testing.T) { + store := ©Store{jobs: map[string]*job.Job{ + "plan": {Status: job.Complete, Output: []string{"plan 1"}}, + }} + handler := newCopyHandler(t, store, ©Registry{}) + + require.NoError(t, handler.CopyJob(context.Background(), "plan", "apply")) + require.NoError(t, store.Write(context.Background(), "apply", "apply 1")) + + assert.Equal(t, []string{"plan 1", job.ApplyDivider, "apply 1"}, store.writeLog) + }) + + t.Run("a missing source copies nothing", func(t *testing.T) { + store := ©Store{jobs: map[string]*job.Job{}} + registry := ©Registry{} + + require.NoError(t, newCopyHandler(t, store, registry).CopyJob(context.Background(), "plan", "apply")) + + assert.Empty(t, store.writeLog) + assert.Empty(t, registry.broadcast) + }) + + t.Run("an empty source copies nothing, not even the divider", func(t *testing.T) { + store := ©Store{jobs: map[string]*job.Job{"plan": {Status: job.Complete}}} + + require.NoError(t, newCopyHandler(t, store, ©Registry{}).CopyJob(context.Background(), "plan", "apply")) + + assert.Empty(t, store.writeLog) + }) + + t.Run("fails when the source can't be read", func(t *testing.T) { + store := ©Store{getErr: errors.New("boom")} + + assert.Error(t, newCopyHandler(t, store, ©Registry{}).CopyJob(context.Background(), "plan", "apply")) + }) + + t.Run("fails when the destination can't be written", func(t *testing.T) { + store := ©Store{ + jobs: map[string]*job.Job{"plan": {Status: job.Complete, Output: []string{"plan 1"}}}, + writeErr: errors.New("boom"), + } + + assert.Error(t, newCopyHandler(t, store, ©Registry{}).CopyJob(context.Background(), "plan", "apply")) + }) +} diff --git a/server/neptune/workflows/activities/job.go b/server/neptune/workflows/activities/job.go index 3e1baa986..e1f7d9940 100644 --- a/server/neptune/workflows/activities/job.go +++ b/server/neptune/workflows/activities/job.go @@ -9,6 +9,7 @@ import ( type closer interface { CloseJob(ctx context.Context, jobID string) error + CopyJob(ctx context.Context, fromJobID string, toJobID string) error } type jobActivities struct { @@ -26,3 +27,14 @@ func (t *jobActivities) CloseJob(ctx context.Context, request CloseJobRequest) e } return err } + +type CopyJobOutputRequest struct { + FromJobID string + ToJobID string +} + +// CopyJobOutput starts ToJobID's log with the output of FromJobID. Callers must not retry it, a second run +// would write the output twice. +func (t *jobActivities) CopyJobOutput(ctx context.Context, request CopyJobOutputRequest) error { + return errors.Wrapf(t.StreamCloser.CopyJob(ctx, request.FromJobID, request.ToJobID), "copying job output") +} diff --git a/server/neptune/workflows/deploy_test.go b/server/neptune/workflows/deploy_test.go index 8b26ec19e..e684d0830 100644 --- a/server/neptune/workflows/deploy_test.go +++ b/server/neptune/workflows/deploy_test.go @@ -232,6 +232,10 @@ func (sc *testStreamCloser) CloseJob(ctx context.Context, jobID string) error { return nil } +func (sc *testStreamCloser) CopyJob(ctx context.Context, fromJobID string, toJobID string) error { + return nil +} + var fileContents = ` resource "null_resource" "null" {} ` diff --git a/server/neptune/workflows/internal/terraform/state/store.go b/server/neptune/workflows/internal/terraform/state/store.go index afcfd13c7..f83f5d07f 100644 --- a/server/neptune/workflows/internal/terraform/state/store.go +++ b/server/neptune/workflows/internal/terraform/state/store.go @@ -61,6 +61,14 @@ func NewWorkflowStore(notifier UpdateNotifier, mode terraform.WorkflowMode, id s return NewWorkflowStoreWithGenerator(notifier, urlGenerator, mode, id) } +// PlanJobID returns the id of the plan job, or empty if no plan job was started. +func (s *WorkflowStore) PlanJobID() string { + if s.state.Plan == nil { + return "" + } + return s.state.Plan.ID +} + func (s *WorkflowStore) InitPlanJob(jobID fmt.Stringer, serverURL fmt.Stringer) error { outputURL, err := s.outputURLGenerator.Generate(jobID, serverURL) diff --git a/server/neptune/workflows/internal/terraform/workflow.go b/server/neptune/workflows/internal/terraform/workflow.go index 6d1d36c23..812adcdd2 100644 --- a/server/neptune/workflows/internal/terraform/workflow.go +++ b/server/neptune/workflows/internal/terraform/workflow.go @@ -31,6 +31,7 @@ type githubActivities interface { type terraformActivities interface { Cleanup(ctx context.Context, request activities.CleanupRequest) (activities.CleanupResponse, error) GetWorkerInfo(ctx context.Context) (*activities.GetWorkerInfoResponse, error) + CopyJobOutput(ctx context.Context, request activities.CopyJobOutputRequest) error } // jobRunner runs a deploy plan/apply job @@ -262,6 +263,8 @@ func (r *Runner) Apply(ctx workflow.Context, root *terraform.LocalRoot, serverUR return newUpdateJobError(err, "unable to update job with success status") } + r.copyPlanOutputToApplyJob(ctx, jobID.String()) + err = r.JobRunner.Apply(ctx, root, jobID.String(), planResponse.PlanFile) if err != nil { if err := r.Store.UpdateApplyJobWithStatus(state.FailedJobStatus, state.UpdateOptions{ @@ -282,6 +285,33 @@ func (r *Runner) Apply(ctx workflow.Context, root *terraform.LocalRoot, serverUR return nil } +// copyPlanOutputToApplyJob starts the apply job's log with the plan's, so a deploy's apply log shows the +// plan and the apply together. This is best effort: the apply must not fail or be delayed because the plan +// output couldn't be copied. +func (r *Runner) copyPlanOutputToApplyJob(ctx workflow.Context, applyJobID string) { + // Only deploy mode gets here: PR and adhoc workflows return before the apply step. + planJobID := r.Store.PlanJobID() + if planJobID == "" { + return + } + + // This adds an activity, so workflows that already got past this point before the change was deployed + // must not run it when they replay. + if workflow.GetVersion(ctx, "CopyPlanOutputToApplyJob", workflow.DefaultVersion, workflow.Version(1)) == workflow.DefaultVersion { + return + } + + // copying twice would write the plan output twice + ctx = workflow.WithRetryPolicy(ctx, temporal.RetryPolicy{MaximumAttempts: 1}) + err := workflow.ExecuteActivity(ctx, r.TerraformActivities.CopyJobOutput, activities.CopyJobOutputRequest{ + FromJobID: planJobID, + ToJobID: applyJobID, + }).Get(ctx, nil) + if err != nil { + workflow.GetLogger(ctx).Error("unable to copy plan output to the apply job", key.ErrKey, err) + } +} + func (r *Runner) Run(ctx workflow.Context) (Response, error) { var err error var resp Response diff --git a/server/neptune/workflows/internal/terraform/workflow_test.go b/server/neptune/workflows/internal/terraform/workflow_test.go index f0fbc97b7..3dd50b2c4 100644 --- a/server/neptune/workflows/internal/terraform/workflow_test.go +++ b/server/neptune/workflows/internal/terraform/workflow_test.go @@ -98,6 +98,9 @@ type terraformActivities struct{} func (a *terraformActivities) Cleanup(ctx context.Context, request activities.CleanupRequest) (activities.CleanupResponse, error) { return activities.CleanupResponse{}, nil } +func (a *terraformActivities) CopyJobOutput(ctx context.Context, request activities.CopyJobOutputRequest) error { + return nil +} func (a *terraformActivities) GetWorkerInfo(ctx context.Context) (*activities.GetWorkerInfoResponse, error) { u, err := url.Parse("www.test.com") return &activities.GetWorkerInfoResponse{ @@ -297,6 +300,11 @@ func TestSuccess_DeployMode(t *testing.T) { DeployDirectory: DeployDir, }).Return(activities.CleanupResponse{}, nil) + // the apply job's log starts with the plan's + env.OnActivity(ta.CopyJobOutput, mock.Anything, mock.MatchedBy(func(req activities.CopyJobOutputRequest) bool { + return req.FromJobID != "" && req.ToJobID != "" && req.FromJobID != req.ToJobID + })).Return(nil).Once() + // send approval of plan env.RegisterDelayedCallback(func() { env.SignalWorkflow("planreview", gate.PlanReviewSignalRequest{ @@ -502,6 +510,9 @@ func TestSuccess_PRMode(t *testing.T) { LocalRoot: testLocalRoot, DeployDirectory: DeployDir, }, nil) + env.OnActivity(ta.CopyJobOutput, mock.Anything, mock.Anything).Return(nil).Run(func(mock.Arguments) { + t.Error("PR mode has no apply job, so no plan output should be copied") + }).Maybe() env.OnActivity(ta.Cleanup, mock.Anything, activities.CleanupRequest{ DeployDirectory: DeployDir, }).Return(activities.CleanupResponse{}, nil) @@ -1136,3 +1147,32 @@ func TestCleanupErrorReturnsNoError(t *testing.T) { err := env.GetWorkflowResult(&resp) assert.NoError(t, err) } + +func TestDeployMode_CopyPlanOutputFailureDoesNotFailApply(t *testing.T) { + var suite testsuite.WorkflowTestSuite + env := suite.NewTestWorkflowEnvironment() + ga := &githubActivities{} + ta := &terraformActivities{} + env.RegisterActivity(ga) + env.RegisterActivity(ta) + + env.OnActivity(ga.GithubFetchRoot, mock.Anything, mock.Anything).Return(activities.FetchRootResponse{ + LocalRoot: testLocalRoot, + DeployDirectory: DeployDir, + }, nil) + env.OnActivity(ta.Cleanup, mock.Anything, mock.Anything).Return(activities.CleanupResponse{}, nil) + env.OnActivity(ta.CopyJobOutput, mock.Anything, mock.Anything).Return(errors.New("copy failed")).Once() + + env.RegisterDelayedCallback(func() { + env.SignalWorkflow("planreview", gate.PlanReviewSignalRequest{ + Status: gate.Approved, + }) + }, 5*time.Second) + + env.ExecuteWorkflow(testTerraformWorkflow, request{}) + assert.True(t, env.IsWorkflowCompleted()) + + var resp response + assert.NoError(t, env.GetWorkflowResult(&resp)) + env.AssertExpectations(t) +}