From 55f62ef23e85d5f0dac1dec71a725683b1635f33 Mon Sep 17 00:00:00 2001 From: Emmanuel Maduwuba Date: Fri, 13 Nov 2026 11:43:00 -0500 Subject: [PATCH 1/3] Start the deploy apply job log with the plan output In deploy mode the plan and apply run as separate jobs with separate logs, and only the apply job's log is persisted under the id that is audited and shown downstream. Before the apply starts, copy the plan job's output into the apply job so its log shows plan then apply. The copy is a new activity gated behind workflow.GetVersion so workflows already past this point replay unchanged. It is best effort and never retried (a retry would write the plan twice): a failure is logged and the apply still runs. --- .../temporalworker/job/stream_handler.go | 26 +++++ .../job/stream_handler_copy_test.go | 107 ++++++++++++++++++ server/neptune/workflows/activities/job.go | 12 ++ server/neptune/workflows/deploy_test.go | 4 + .../internal/terraform/state/store.go | 8 ++ .../workflows/internal/terraform/workflow.go | 30 +++++ .../internal/terraform/workflow_test.go | 40 +++++++ 7 files changed, 227 insertions(+) create mode 100644 server/neptune/temporalworker/job/stream_handler_copy_test.go diff --git a/server/neptune/temporalworker/job/stream_handler.go b/server/neptune/temporalworker/job/stream_handler.go index dbdb80d12..cb23283f1 100644 --- a/server/neptune/temporalworker/job/stream_handler.go +++ b/server/neptune/temporalworker/job/stream_handler.go @@ -5,6 +5,7 @@ import ( "fmt" "sync" + "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/config/valid" "github.com/runatlantis/atlantis/server/legacy/events/terraform/filter" "github.com/runatlantis/atlantis/server/logging" @@ -78,6 +79,31 @@ func (s *StreamHandler) CloseJob(ctx context.Context, jobID string) error { return s.Store.Close(ctx, jobID, Complete) } +// CopyJob appends the output recorded for fromJobID to toJobID, ahead of anything toJobID writes afterwards. +// It is used to start an apply job's log with the plan it applies, so the apply log (which is what gets +// persisted and picked up downstream) shows plan and apply together. +// +// It is not idempotent: calling it again appends the lines again, so callers must not retry it. +func (s *StreamHandler) CopyJob(ctx context.Context, fromJobID string, toJobID string) error { + from, err := s.Store.Get(ctx, fromJobID) + if err != nil { + return errors.Wrapf(err, "getting job %s", fromJobID) + } + if from == nil { + return nil + } + + // copy the slice header's contents up front: the source is complete but may still be shared with the store + lines := append([]string(nil), from.Output...) + for _, line := range lines { + s.ReceiverRegistry.Broadcast(OutputLine{JobID: toJobID, Line: line}) + if err := s.Store.Write(ctx, toJobID, line); err != nil { + return errors.Wrapf(err, "writing to job %s", toJobID) + } + } + return nil +} + func (s *StreamHandler) CleanUp(ctx context.Context) error { s.Logger.InfoContext(ctx, "waiting for jobs to finish") // Should we timeout here? We do risk a deadlock without this however adding a timeout also risks diff --git a/server/neptune/temporalworker/job/stream_handler_copy_test.go b/server/neptune/temporalworker/job/stream_handler_copy_test.go new file mode 100644 index 000000000..ca4474ab0 --- /dev/null +++ b/server/neptune/temporalworker/job/stream_handler_copy_test.go @@ -0,0 +1,107 @@ +package job_test + +import ( + "context" + "errors" + "testing" + + "github.com/runatlantis/atlantis/server/config/valid" + "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/neptune/temporalworker/job" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type copyStore struct { + jobs map[string]*job.Job + getErr error + writeErr error + writeLog []string + writeKeys []string +} + +func (s *copyStore) Get(_ context.Context, jobID string) (*job.Job, error) { + if s.getErr != nil { + return nil, s.getErr + } + return s.jobs[jobID], nil +} + +func (s *copyStore) Write(_ context.Context, jobID string, output string) error { + if s.writeErr != nil { + return s.writeErr + } + s.writeKeys = append(s.writeKeys, jobID) + s.writeLog = append(s.writeLog, output) + return nil +} + +func (s *copyStore) Remove(string) {} +func (s *copyStore) Close(context.Context, string, job.JobStatus) error { return nil } +func (s *copyStore) Cleanup(context.Context) error { return nil } + +type copyRegistry struct { + broadcast []job.OutputLine +} + +func (r *copyRegistry) AddReceiver(string, chan string) {} +func (r *copyRegistry) Broadcast(msg job.OutputLine) { r.broadcast = append(r.broadcast, msg) } +func (r *copyRegistry) CleanUp() {} +func (r *copyRegistry) Close(context.Context, string) {} + +func newCopyHandler(t *testing.T, store *copyStore, registry *copyRegistry) *job.StreamHandler { + t.Helper() + return job.NewStreamHandler(store, registry, valid.TerraformLogFilters{}, logging.NewNoopCtxLogger(t)) +} + +func TestStreamHandler_CopyJob(t *testing.T) { + t.Run("writes the source output to the destination in order and broadcasts it", func(t *testing.T) { + store := ©Store{jobs: map[string]*job.Job{ + "plan": {Status: job.Complete, Output: []string{"plan 1", "plan 2"}}, + }} + registry := ©Registry{} + + require.NoError(t, newCopyHandler(t, store, registry).CopyJob(context.Background(), "plan", "apply")) + + assert.Equal(t, []string{"plan 1", "plan 2"}, store.writeLog) + assert.Equal(t, []string{"apply", "apply"}, store.writeKeys) + assert.Equal(t, []job.OutputLine{{JobID: "apply", Line: "plan 1"}, {JobID: "apply", Line: "plan 2"}}, registry.broadcast) + }) + + t.Run("later apply output follows the copied plan", func(t *testing.T) { + store := ©Store{jobs: map[string]*job.Job{ + "plan": {Status: job.Complete, Output: []string{"plan 1"}}, + }} + handler := newCopyHandler(t, store, ©Registry{}) + + require.NoError(t, handler.CopyJob(context.Background(), "plan", "apply")) + require.NoError(t, store.Write(context.Background(), "apply", "apply 1")) + + assert.Equal(t, []string{"plan 1", "apply 1"}, store.writeLog) + }) + + t.Run("a missing source copies nothing", func(t *testing.T) { + store := ©Store{jobs: map[string]*job.Job{}} + registry := ©Registry{} + + require.NoError(t, newCopyHandler(t, store, registry).CopyJob(context.Background(), "plan", "apply")) + + assert.Empty(t, store.writeLog) + assert.Empty(t, registry.broadcast) + }) + + t.Run("fails when the source can't be read", func(t *testing.T) { + store := ©Store{getErr: errors.New("boom")} + + assert.Error(t, newCopyHandler(t, store, ©Registry{}).CopyJob(context.Background(), "plan", "apply")) + }) + + t.Run("fails when the destination can't be written", func(t *testing.T) { + store := ©Store{ + jobs: map[string]*job.Job{"plan": {Status: job.Complete, Output: []string{"plan 1"}}}, + writeErr: errors.New("boom"), + } + + assert.Error(t, newCopyHandler(t, store, ©Registry{}).CopyJob(context.Background(), "plan", "apply")) + }) +} diff --git a/server/neptune/workflows/activities/job.go b/server/neptune/workflows/activities/job.go index 3e1baa986..e1f7d9940 100644 --- a/server/neptune/workflows/activities/job.go +++ b/server/neptune/workflows/activities/job.go @@ -9,6 +9,7 @@ import ( type closer interface { CloseJob(ctx context.Context, jobID string) error + CopyJob(ctx context.Context, fromJobID string, toJobID string) error } type jobActivities struct { @@ -26,3 +27,14 @@ func (t *jobActivities) CloseJob(ctx context.Context, request CloseJobRequest) e } return err } + +type CopyJobOutputRequest struct { + FromJobID string + ToJobID string +} + +// CopyJobOutput starts ToJobID's log with the output of FromJobID. Callers must not retry it, a second run +// would write the output twice. +func (t *jobActivities) CopyJobOutput(ctx context.Context, request CopyJobOutputRequest) error { + return errors.Wrapf(t.StreamCloser.CopyJob(ctx, request.FromJobID, request.ToJobID), "copying job output") +} diff --git a/server/neptune/workflows/deploy_test.go b/server/neptune/workflows/deploy_test.go index 8b26ec19e..e684d0830 100644 --- a/server/neptune/workflows/deploy_test.go +++ b/server/neptune/workflows/deploy_test.go @@ -232,6 +232,10 @@ func (sc *testStreamCloser) CloseJob(ctx context.Context, jobID string) error { return nil } +func (sc *testStreamCloser) CopyJob(ctx context.Context, fromJobID string, toJobID string) error { + return nil +} + var fileContents = ` resource "null_resource" "null" {} ` diff --git a/server/neptune/workflows/internal/terraform/state/store.go b/server/neptune/workflows/internal/terraform/state/store.go index afcfd13c7..f83f5d07f 100644 --- a/server/neptune/workflows/internal/terraform/state/store.go +++ b/server/neptune/workflows/internal/terraform/state/store.go @@ -61,6 +61,14 @@ func NewWorkflowStore(notifier UpdateNotifier, mode terraform.WorkflowMode, id s return NewWorkflowStoreWithGenerator(notifier, urlGenerator, mode, id) } +// PlanJobID returns the id of the plan job, or empty if no plan job was started. +func (s *WorkflowStore) PlanJobID() string { + if s.state.Plan == nil { + return "" + } + return s.state.Plan.ID +} + func (s *WorkflowStore) InitPlanJob(jobID fmt.Stringer, serverURL fmt.Stringer) error { outputURL, err := s.outputURLGenerator.Generate(jobID, serverURL) diff --git a/server/neptune/workflows/internal/terraform/workflow.go b/server/neptune/workflows/internal/terraform/workflow.go index 6d1d36c23..812adcdd2 100644 --- a/server/neptune/workflows/internal/terraform/workflow.go +++ b/server/neptune/workflows/internal/terraform/workflow.go @@ -31,6 +31,7 @@ type githubActivities interface { type terraformActivities interface { Cleanup(ctx context.Context, request activities.CleanupRequest) (activities.CleanupResponse, error) GetWorkerInfo(ctx context.Context) (*activities.GetWorkerInfoResponse, error) + CopyJobOutput(ctx context.Context, request activities.CopyJobOutputRequest) error } // jobRunner runs a deploy plan/apply job @@ -262,6 +263,8 @@ func (r *Runner) Apply(ctx workflow.Context, root *terraform.LocalRoot, serverUR return newUpdateJobError(err, "unable to update job with success status") } + r.copyPlanOutputToApplyJob(ctx, jobID.String()) + err = r.JobRunner.Apply(ctx, root, jobID.String(), planResponse.PlanFile) if err != nil { if err := r.Store.UpdateApplyJobWithStatus(state.FailedJobStatus, state.UpdateOptions{ @@ -282,6 +285,33 @@ func (r *Runner) Apply(ctx workflow.Context, root *terraform.LocalRoot, serverUR return nil } +// copyPlanOutputToApplyJob starts the apply job's log with the plan's, so a deploy's apply log shows the +// plan and the apply together. This is best effort: the apply must not fail or be delayed because the plan +// output couldn't be copied. +func (r *Runner) copyPlanOutputToApplyJob(ctx workflow.Context, applyJobID string) { + // Only deploy mode gets here: PR and adhoc workflows return before the apply step. + planJobID := r.Store.PlanJobID() + if planJobID == "" { + return + } + + // This adds an activity, so workflows that already got past this point before the change was deployed + // must not run it when they replay. + if workflow.GetVersion(ctx, "CopyPlanOutputToApplyJob", workflow.DefaultVersion, workflow.Version(1)) == workflow.DefaultVersion { + return + } + + // copying twice would write the plan output twice + ctx = workflow.WithRetryPolicy(ctx, temporal.RetryPolicy{MaximumAttempts: 1}) + err := workflow.ExecuteActivity(ctx, r.TerraformActivities.CopyJobOutput, activities.CopyJobOutputRequest{ + FromJobID: planJobID, + ToJobID: applyJobID, + }).Get(ctx, nil) + if err != nil { + workflow.GetLogger(ctx).Error("unable to copy plan output to the apply job", key.ErrKey, err) + } +} + func (r *Runner) Run(ctx workflow.Context) (Response, error) { var err error var resp Response diff --git a/server/neptune/workflows/internal/terraform/workflow_test.go b/server/neptune/workflows/internal/terraform/workflow_test.go index f0fbc97b7..3dd50b2c4 100644 --- a/server/neptune/workflows/internal/terraform/workflow_test.go +++ b/server/neptune/workflows/internal/terraform/workflow_test.go @@ -98,6 +98,9 @@ type terraformActivities struct{} func (a *terraformActivities) Cleanup(ctx context.Context, request activities.CleanupRequest) (activities.CleanupResponse, error) { return activities.CleanupResponse{}, nil } +func (a *terraformActivities) CopyJobOutput(ctx context.Context, request activities.CopyJobOutputRequest) error { + return nil +} func (a *terraformActivities) GetWorkerInfo(ctx context.Context) (*activities.GetWorkerInfoResponse, error) { u, err := url.Parse("www.test.com") return &activities.GetWorkerInfoResponse{ @@ -297,6 +300,11 @@ func TestSuccess_DeployMode(t *testing.T) { DeployDirectory: DeployDir, }).Return(activities.CleanupResponse{}, nil) + // the apply job's log starts with the plan's + env.OnActivity(ta.CopyJobOutput, mock.Anything, mock.MatchedBy(func(req activities.CopyJobOutputRequest) bool { + return req.FromJobID != "" && req.ToJobID != "" && req.FromJobID != req.ToJobID + })).Return(nil).Once() + // send approval of plan env.RegisterDelayedCallback(func() { env.SignalWorkflow("planreview", gate.PlanReviewSignalRequest{ @@ -502,6 +510,9 @@ func TestSuccess_PRMode(t *testing.T) { LocalRoot: testLocalRoot, DeployDirectory: DeployDir, }, nil) + env.OnActivity(ta.CopyJobOutput, mock.Anything, mock.Anything).Return(nil).Run(func(mock.Arguments) { + t.Error("PR mode has no apply job, so no plan output should be copied") + }).Maybe() env.OnActivity(ta.Cleanup, mock.Anything, activities.CleanupRequest{ DeployDirectory: DeployDir, }).Return(activities.CleanupResponse{}, nil) @@ -1136,3 +1147,32 @@ func TestCleanupErrorReturnsNoError(t *testing.T) { err := env.GetWorkflowResult(&resp) assert.NoError(t, err) } + +func TestDeployMode_CopyPlanOutputFailureDoesNotFailApply(t *testing.T) { + var suite testsuite.WorkflowTestSuite + env := suite.NewTestWorkflowEnvironment() + ga := &githubActivities{} + ta := &terraformActivities{} + env.RegisterActivity(ga) + env.RegisterActivity(ta) + + env.OnActivity(ga.GithubFetchRoot, mock.Anything, mock.Anything).Return(activities.FetchRootResponse{ + LocalRoot: testLocalRoot, + DeployDirectory: DeployDir, + }, nil) + env.OnActivity(ta.Cleanup, mock.Anything, mock.Anything).Return(activities.CleanupResponse{}, nil) + env.OnActivity(ta.CopyJobOutput, mock.Anything, mock.Anything).Return(errors.New("copy failed")).Once() + + env.RegisterDelayedCallback(func() { + env.SignalWorkflow("planreview", gate.PlanReviewSignalRequest{ + Status: gate.Approved, + }) + }, 5*time.Second) + + env.ExecuteWorkflow(testTerraformWorkflow, request{}) + assert.True(t, env.IsWorkflowCompleted()) + + var resp response + assert.NoError(t, env.GetWorkflowResult(&resp)) + env.AssertExpectations(t) +} From ef1f6085d6981515109c6a2e70cbd97f2853d90b Mon Sep 17 00:00:00 2001 From: Emmanuel Maduwuba Date: Fri, 13 Nov 2026 12:15:42 -0500 Subject: [PATCH 2/3] Add a divider between the copied plan and the apply output So readers of the combined apply log can see where the plan ends and the apply begins. Nothing is written when there is no plan output. --- .../temporalworker/job/stream_handler.go | 14 ++++++++----- .../job/stream_handler_copy_test.go | 20 +++++++++++++++---- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/server/neptune/temporalworker/job/stream_handler.go b/server/neptune/temporalworker/job/stream_handler.go index cb23283f1..bd2e04911 100644 --- a/server/neptune/temporalworker/job/stream_handler.go +++ b/server/neptune/temporalworker/job/stream_handler.go @@ -79,9 +79,12 @@ func (s *StreamHandler) CloseJob(ctx context.Context, jobID string) error { return s.Store.Close(ctx, jobID, Complete) } -// CopyJob appends the output recorded for fromJobID to toJobID, ahead of anything toJobID writes afterwards. -// It is used to start an apply job's log with the plan it applies, so the apply log (which is what gets -// persisted and picked up downstream) shows plan and apply together. +// ApplyDivider is written between the copied plan output and the apply's own output. +const ApplyDivider = "---- terraform apply ----" + +// CopyJob appends the output recorded for fromJobID to toJobID, followed by ApplyDivider, ahead of anything +// toJobID writes afterwards. It is used to start an apply job's log with the plan it applies, so the apply log +// (which is what gets persisted and picked up downstream) shows plan and apply together. // // It is not idempotent: calling it again appends the lines again, so callers must not retry it. func (s *StreamHandler) CopyJob(ctx context.Context, fromJobID string, toJobID string) error { @@ -89,12 +92,13 @@ func (s *StreamHandler) CopyJob(ctx context.Context, fromJobID string, toJobID s if err != nil { return errors.Wrapf(err, "getting job %s", fromJobID) } - if from == nil { + if from == nil || len(from.Output) == 0 { return nil } - // copy the slice header's contents up front: the source is complete but may still be shared with the store + // copy the lines up front: the source is complete but may still be shared with the store lines := append([]string(nil), from.Output...) + lines = append(lines, ApplyDivider) for _, line := range lines { s.ReceiverRegistry.Broadcast(OutputLine{JobID: toJobID, Line: line}) if err := s.Store.Write(ctx, toJobID, line); err != nil { diff --git a/server/neptune/temporalworker/job/stream_handler_copy_test.go b/server/neptune/temporalworker/job/stream_handler_copy_test.go index ca4474ab0..a915210eb 100644 --- a/server/neptune/temporalworker/job/stream_handler_copy_test.go +++ b/server/neptune/temporalworker/job/stream_handler_copy_test.go @@ -63,9 +63,13 @@ func TestStreamHandler_CopyJob(t *testing.T) { require.NoError(t, newCopyHandler(t, store, registry).CopyJob(context.Background(), "plan", "apply")) - assert.Equal(t, []string{"plan 1", "plan 2"}, store.writeLog) - assert.Equal(t, []string{"apply", "apply"}, store.writeKeys) - assert.Equal(t, []job.OutputLine{{JobID: "apply", Line: "plan 1"}, {JobID: "apply", Line: "plan 2"}}, registry.broadcast) + assert.Equal(t, []string{"plan 1", "plan 2", job.ApplyDivider}, store.writeLog) + assert.Equal(t, []string{"apply", "apply", "apply"}, store.writeKeys) + assert.Equal(t, []job.OutputLine{ + {JobID: "apply", Line: "plan 1"}, + {JobID: "apply", Line: "plan 2"}, + {JobID: "apply", Line: job.ApplyDivider}, + }, registry.broadcast) }) t.Run("later apply output follows the copied plan", func(t *testing.T) { @@ -77,7 +81,7 @@ func TestStreamHandler_CopyJob(t *testing.T) { require.NoError(t, handler.CopyJob(context.Background(), "plan", "apply")) require.NoError(t, store.Write(context.Background(), "apply", "apply 1")) - assert.Equal(t, []string{"plan 1", "apply 1"}, store.writeLog) + assert.Equal(t, []string{"plan 1", job.ApplyDivider, "apply 1"}, store.writeLog) }) t.Run("a missing source copies nothing", func(t *testing.T) { @@ -90,6 +94,14 @@ func TestStreamHandler_CopyJob(t *testing.T) { assert.Empty(t, registry.broadcast) }) + t.Run("an empty source copies nothing, not even the divider", func(t *testing.T) { + store := ©Store{jobs: map[string]*job.Job{"plan": {Status: job.Complete}}} + + require.NoError(t, newCopyHandler(t, store, ©Registry{}).CopyJob(context.Background(), "plan", "apply")) + + assert.Empty(t, store.writeLog) + }) + t.Run("fails when the source can't be read", func(t *testing.T) { store := ©Store{getErr: errors.New("boom")} From e283c58d39363f6293b5eddbba0323af58ae65fa Mon Sep 17 00:00:00 2001 From: Emmanuel Maduwuba Date: Fri, 13 Nov 2026 12:31:18 -0500 Subject: [PATCH 3/3] Fix a hang in the receiver registry broadcast test Broadcast drops a receiver that isn't ready to receive, so the test's unbuffered channel made it wait forever whenever Broadcast ran before the read. Buffer the channel so the message is not dropped. --- server/neptune/temporalworker/job/receiver_registry_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/neptune/temporalworker/job/receiver_registry_test.go b/server/neptune/temporalworker/job/receiver_registry_test.go index 0ebdcebc6..c1b71c1f5 100644 --- a/server/neptune/temporalworker/job/receiver_registry_test.go +++ b/server/neptune/temporalworker/job/receiver_registry_test.go @@ -89,7 +89,9 @@ func TestReceiverRegistry(t *testing.T) { t.Run("adds a receiver and broadcast", func(t *testing.T) { recvRegistry := job.NewReceiverRegistry() - ch := make(chan string) + // Broadcast doesn't wait for a receiver (it drops one that isn't ready), so with an unbuffered + // channel the test hangs whenever Broadcast runs before the read below. + ch := make(chan string, 1) recvRegistry.AddReceiver(jobID, ch) go recvRegistry.Broadcast(job.OutputLine{