Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions server/neptune/temporalworker/job/stream_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"sync"

"github.com/pkg/errors"
"github.com/runatlantis/atlantis/server/config/valid"
"github.com/runatlantis/atlantis/server/legacy/events/terraform/filter"
"github.com/runatlantis/atlantis/server/logging"
Expand Down Expand Up @@ -78,6 +79,35 @@ func (s *StreamHandler) CloseJob(ctx context.Context, jobID string) error {
return s.Store.Close(ctx, jobID, Complete)
}

// ApplyDivider is written between the copied plan output and the apply's own output.
const ApplyDivider = "---- terraform apply ----"

// CopyJob appends the output recorded for fromJobID to toJobID, followed by ApplyDivider, ahead of anything
// toJobID writes afterwards. It is used to start an apply job's log with the plan it applies, so the apply log
// (which is what gets persisted and picked up downstream) shows plan and apply together.
//
// It is not idempotent: calling it again appends the lines again, so callers must not retry it.
func (s *StreamHandler) CopyJob(ctx context.Context, fromJobID string, toJobID string) error {
from, err := s.Store.Get(ctx, fromJobID)
if err != nil {
return errors.Wrapf(err, "getting job %s", fromJobID)
}
if from == nil || len(from.Output) == 0 {
return nil
}

// copy the lines up front: the source is complete but may still be shared with the store
lines := append([]string(nil), from.Output...)
lines = append(lines, ApplyDivider)
for _, line := range lines {
s.ReceiverRegistry.Broadcast(OutputLine{JobID: toJobID, Line: line})
if err := s.Store.Write(ctx, toJobID, line); err != nil {
return errors.Wrapf(err, "writing to job %s", toJobID)
}
}
return nil
}

func (s *StreamHandler) CleanUp(ctx context.Context) error {
s.Logger.InfoContext(ctx, "waiting for jobs to finish")
// Should we timeout here? We do risk a deadlock without this however adding a timeout also risks
Expand Down
119 changes: 119 additions & 0 deletions server/neptune/temporalworker/job/stream_handler_copy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
package job_test

import (
"context"
"errors"
"testing"

"github.com/runatlantis/atlantis/server/config/valid"
"github.com/runatlantis/atlantis/server/logging"
"github.com/runatlantis/atlantis/server/neptune/temporalworker/job"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

type copyStore struct {
jobs map[string]*job.Job
getErr error
writeErr error
writeLog []string
writeKeys []string
}

func (s *copyStore) Get(_ context.Context, jobID string) (*job.Job, error) {
if s.getErr != nil {
return nil, s.getErr
}
return s.jobs[jobID], nil
}

func (s *copyStore) Write(_ context.Context, jobID string, output string) error {
if s.writeErr != nil {
return s.writeErr
}
s.writeKeys = append(s.writeKeys, jobID)
s.writeLog = append(s.writeLog, output)
return nil
}

func (s *copyStore) Remove(string) {}
func (s *copyStore) Close(context.Context, string, job.JobStatus) error { return nil }
func (s *copyStore) Cleanup(context.Context) error { return nil }

type copyRegistry struct {
broadcast []job.OutputLine
}

func (r *copyRegistry) AddReceiver(string, chan string) {}
func (r *copyRegistry) Broadcast(msg job.OutputLine) { r.broadcast = append(r.broadcast, msg) }
func (r *copyRegistry) CleanUp() {}
func (r *copyRegistry) Close(context.Context, string) {}

func newCopyHandler(t *testing.T, store *copyStore, registry *copyRegistry) *job.StreamHandler {
t.Helper()
return job.NewStreamHandler(store, registry, valid.TerraformLogFilters{}, logging.NewNoopCtxLogger(t))
}

func TestStreamHandler_CopyJob(t *testing.T) {
t.Run("writes the source output to the destination in order and broadcasts it", func(t *testing.T) {
store := &copyStore{jobs: map[string]*job.Job{
"plan": {Status: job.Complete, Output: []string{"plan 1", "plan 2"}},
}}
registry := &copyRegistry{}

require.NoError(t, newCopyHandler(t, store, registry).CopyJob(context.Background(), "plan", "apply"))

assert.Equal(t, []string{"plan 1", "plan 2", job.ApplyDivider}, store.writeLog)
assert.Equal(t, []string{"apply", "apply", "apply"}, store.writeKeys)
assert.Equal(t, []job.OutputLine{
{JobID: "apply", Line: "plan 1"},
{JobID: "apply", Line: "plan 2"},
{JobID: "apply", Line: job.ApplyDivider},
}, registry.broadcast)
})

t.Run("later apply output follows the copied plan", func(t *testing.T) {
store := &copyStore{jobs: map[string]*job.Job{
"plan": {Status: job.Complete, Output: []string{"plan 1"}},
}}
handler := newCopyHandler(t, store, &copyRegistry{})

require.NoError(t, handler.CopyJob(context.Background(), "plan", "apply"))
require.NoError(t, store.Write(context.Background(), "apply", "apply 1"))

assert.Equal(t, []string{"plan 1", job.ApplyDivider, "apply 1"}, store.writeLog)
})

t.Run("a missing source copies nothing", func(t *testing.T) {
store := &copyStore{jobs: map[string]*job.Job{}}
registry := &copyRegistry{}

require.NoError(t, newCopyHandler(t, store, registry).CopyJob(context.Background(), "plan", "apply"))

assert.Empty(t, store.writeLog)
assert.Empty(t, registry.broadcast)
})

t.Run("an empty source copies nothing, not even the divider", func(t *testing.T) {
store := &copyStore{jobs: map[string]*job.Job{"plan": {Status: job.Complete}}}

require.NoError(t, newCopyHandler(t, store, &copyRegistry{}).CopyJob(context.Background(), "plan", "apply"))

assert.Empty(t, store.writeLog)
})

t.Run("fails when the source can't be read", func(t *testing.T) {
store := &copyStore{getErr: errors.New("boom")}

assert.Error(t, newCopyHandler(t, store, &copyRegistry{}).CopyJob(context.Background(), "plan", "apply"))
})

t.Run("fails when the destination can't be written", func(t *testing.T) {
store := &copyStore{
jobs: map[string]*job.Job{"plan": {Status: job.Complete, Output: []string{"plan 1"}}},
writeErr: errors.New("boom"),
}

assert.Error(t, newCopyHandler(t, store, &copyRegistry{}).CopyJob(context.Background(), "plan", "apply"))
})
}
12 changes: 12 additions & 0 deletions server/neptune/workflows/activities/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

type closer interface {
CloseJob(ctx context.Context, jobID string) error
CopyJob(ctx context.Context, fromJobID string, toJobID string) error
}

type jobActivities struct {
Expand All @@ -26,3 +27,14 @@ func (t *jobActivities) CloseJob(ctx context.Context, request CloseJobRequest) e
}
return err
}

type CopyJobOutputRequest struct {
FromJobID string
ToJobID string
}

// CopyJobOutput starts ToJobID's log with the output of FromJobID. Callers must not retry it, a second run
// would write the output twice.
func (t *jobActivities) CopyJobOutput(ctx context.Context, request CopyJobOutputRequest) error {
return errors.Wrapf(t.StreamCloser.CopyJob(ctx, request.FromJobID, request.ToJobID), "copying job output")
}
4 changes: 4 additions & 0 deletions server/neptune/workflows/deploy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,10 @@ func (sc *testStreamCloser) CloseJob(ctx context.Context, jobID string) error {
return nil
}

func (sc *testStreamCloser) CopyJob(ctx context.Context, fromJobID string, toJobID string) error {
return nil
}

var fileContents = ` resource "null_resource" "null" {}
`

Expand Down
8 changes: 8 additions & 0 deletions server/neptune/workflows/internal/terraform/state/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,14 @@ func NewWorkflowStore(notifier UpdateNotifier, mode terraform.WorkflowMode, id s
return NewWorkflowStoreWithGenerator(notifier, urlGenerator, mode, id)
}

// PlanJobID returns the id of the plan job, or empty if no plan job was started.
func (s *WorkflowStore) PlanJobID() string {
if s.state.Plan == nil {
return ""
}
return s.state.Plan.ID
}

func (s *WorkflowStore) InitPlanJob(jobID fmt.Stringer, serverURL fmt.Stringer) error {
outputURL, err := s.outputURLGenerator.Generate(jobID, serverURL)

Expand Down
30 changes: 30 additions & 0 deletions server/neptune/workflows/internal/terraform/workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type githubActivities interface {
type terraformActivities interface {
Cleanup(ctx context.Context, request activities.CleanupRequest) (activities.CleanupResponse, error)
GetWorkerInfo(ctx context.Context) (*activities.GetWorkerInfoResponse, error)
CopyJobOutput(ctx context.Context, request activities.CopyJobOutputRequest) error
}

// jobRunner runs a deploy plan/apply job
Expand Down Expand Up @@ -262,6 +263,8 @@ func (r *Runner) Apply(ctx workflow.Context, root *terraform.LocalRoot, serverUR
return newUpdateJobError(err, "unable to update job with success status")
}

r.copyPlanOutputToApplyJob(ctx, jobID.String())

err = r.JobRunner.Apply(ctx, root, jobID.String(), planResponse.PlanFile)
if err != nil {
if err := r.Store.UpdateApplyJobWithStatus(state.FailedJobStatus, state.UpdateOptions{
Expand All @@ -282,6 +285,33 @@ func (r *Runner) Apply(ctx workflow.Context, root *terraform.LocalRoot, serverUR
return nil
}

// copyPlanOutputToApplyJob starts the apply job's log with the plan's, so a deploy's apply log shows the
// plan and the apply together. This is best effort: the apply must not fail or be delayed because the plan
// output couldn't be copied.
func (r *Runner) copyPlanOutputToApplyJob(ctx workflow.Context, applyJobID string) {
// Only deploy mode gets here: PR and adhoc workflows return before the apply step.
planJobID := r.Store.PlanJobID()
if planJobID == "" {
return
}

// This adds an activity, so workflows that already got past this point before the change was deployed
// must not run it when they replay.
if workflow.GetVersion(ctx, "CopyPlanOutputToApplyJob", workflow.DefaultVersion, workflow.Version(1)) == workflow.DefaultVersion {
return
}

// copying twice would write the plan output twice
ctx = workflow.WithRetryPolicy(ctx, temporal.RetryPolicy{MaximumAttempts: 1})
err := workflow.ExecuteActivity(ctx, r.TerraformActivities.CopyJobOutput, activities.CopyJobOutputRequest{
FromJobID: planJobID,
ToJobID: applyJobID,
}).Get(ctx, nil)
if err != nil {
workflow.GetLogger(ctx).Error("unable to copy plan output to the apply job", key.ErrKey, err)
}
}

func (r *Runner) Run(ctx workflow.Context) (Response, error) {
var err error
var resp Response
Expand Down
40 changes: 40 additions & 0 deletions server/neptune/workflows/internal/terraform/workflow_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ type terraformActivities struct{}
func (a *terraformActivities) Cleanup(ctx context.Context, request activities.CleanupRequest) (activities.CleanupResponse, error) {
return activities.CleanupResponse{}, nil
}
func (a *terraformActivities) CopyJobOutput(ctx context.Context, request activities.CopyJobOutputRequest) error {
return nil
}
func (a *terraformActivities) GetWorkerInfo(ctx context.Context) (*activities.GetWorkerInfoResponse, error) {
u, err := url.Parse("www.test.com")
return &activities.GetWorkerInfoResponse{
Expand Down Expand Up @@ -297,6 +300,11 @@ func TestSuccess_DeployMode(t *testing.T) {
DeployDirectory: DeployDir,
}).Return(activities.CleanupResponse{}, nil)

// the apply job's log starts with the plan's
env.OnActivity(ta.CopyJobOutput, mock.Anything, mock.MatchedBy(func(req activities.CopyJobOutputRequest) bool {
return req.FromJobID != "" && req.ToJobID != "" && req.FromJobID != req.ToJobID
})).Return(nil).Once()

// send approval of plan
env.RegisterDelayedCallback(func() {
env.SignalWorkflow("planreview", gate.PlanReviewSignalRequest{
Expand Down Expand Up @@ -502,6 +510,9 @@ func TestSuccess_PRMode(t *testing.T) {
LocalRoot: testLocalRoot,
DeployDirectory: DeployDir,
}, nil)
env.OnActivity(ta.CopyJobOutput, mock.Anything, mock.Anything).Return(nil).Run(func(mock.Arguments) {
t.Error("PR mode has no apply job, so no plan output should be copied")
}).Maybe()
env.OnActivity(ta.Cleanup, mock.Anything, activities.CleanupRequest{
DeployDirectory: DeployDir,
}).Return(activities.CleanupResponse{}, nil)
Expand Down Expand Up @@ -1136,3 +1147,32 @@ func TestCleanupErrorReturnsNoError(t *testing.T) {
err := env.GetWorkflowResult(&resp)
assert.NoError(t, err)
}

func TestDeployMode_CopyPlanOutputFailureDoesNotFailApply(t *testing.T) {
var suite testsuite.WorkflowTestSuite
env := suite.NewTestWorkflowEnvironment()
ga := &githubActivities{}
ta := &terraformActivities{}
env.RegisterActivity(ga)
env.RegisterActivity(ta)

env.OnActivity(ga.GithubFetchRoot, mock.Anything, mock.Anything).Return(activities.FetchRootResponse{
LocalRoot: testLocalRoot,
DeployDirectory: DeployDir,
}, nil)
env.OnActivity(ta.Cleanup, mock.Anything, mock.Anything).Return(activities.CleanupResponse{}, nil)
env.OnActivity(ta.CopyJobOutput, mock.Anything, mock.Anything).Return(errors.New("copy failed")).Once()

env.RegisterDelayedCallback(func() {
env.SignalWorkflow("planreview", gate.PlanReviewSignalRequest{
Status: gate.Approved,
})
}, 5*time.Second)

env.ExecuteWorkflow(testTerraformWorkflow, request{})
assert.True(t, env.IsWorkflowCompleted())

var resp response
assert.NoError(t, env.GetWorkflowResult(&resp))
env.AssertExpectations(t)
}
Loading