Skip to content

Commit b156742

Browse files
authored
feat(interlink): patch stage support for foreign executions (#3520)
* feat(interlink): patch stage support for foreign executions Some operations like skip wait or manual approvals result in calls to PATCH /pipelines/pipelineId/stages/stageId with a context map values used to augment the designated stage context. The TaskController also updates the stage's `lastModified` field with user and account information, so that manual approval stages can be used to propagate auth info. This is all great as long as we operate on local executions, but for foreign executions (owned by a different orca cluster), we can't just patch read-only stages. To support this, we: - make `ExecutionRepository::storeStage` foreign-aware - for foreign executions, we create a `PatchStageInterlinkEvent` (if supported) that contains the body of the stage - the stage body includes the updated context map values and `lastModified` information - we add an `updateStage` method in `CompoundExecutionOperator` and update the `TaskController` to call it - we also call it from the event's `applyTo` method * chore(interlink): simplify a few parts - using stageById instead of the loop-based lookup - use @DaTa on LastModifiedDetails - do the time check in StageExecution::setLastModified, but only warn (i.e. be more permissive) - adjust test behavior accordingly * fix(interlink): do not run queue action if repo action failed
1 parent 7ab5f5e commit b156742

12 files changed

Lines changed: 421 additions & 139 deletions

File tree

‎orca/orca-api/src/main/java/com/netflix/spinnaker/orca/api/pipeline/models/StageExecution.java‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
import java.util.function.Predicate;
2929
import javax.annotation.Nonnull;
3030
import javax.annotation.Nullable;
31+
import lombok.Data;
32+
import lombok.NoArgsConstructor;
33+
import lombok.NonNull;
3134

3235
/** The runtime execution state of a stage. */
3336
@Beta
@@ -183,21 +186,15 @@ public interface StageExecution {
183186
@Nonnull
184187
List<StageExecution> downstreamStages();
185188

189+
@Data
190+
@NoArgsConstructor
186191
class LastModifiedDetails implements Serializable {
192+
@NonNull private String user;
187193

188-
private String user;
189-
private Collection<String> allowedAccounts = emptySet();
194+
@NonNull private Collection<String> allowedAccounts = emptySet();
190195

191196
/** TODO(rz): Convert to Instant */
192-
private Long lastModifiedTime;
193-
194-
public @Nonnull String getUser() {
195-
return user;
196-
}
197-
198-
public void setUser(@Nonnull String user) {
199-
this.user = user;
200-
}
197+
@NonNull private Long lastModifiedTime;
201198

202199
public @Nonnull Collection<String> getAllowedAccounts() {
203200
return ImmutableSet.copyOf(allowedAccounts);
@@ -206,13 +203,5 @@ public void setUser(@Nonnull String user) {
206203
public void setAllowedAccounts(@Nonnull Collection<String> allowedAccounts) {
207204
this.allowedAccounts = ImmutableSet.copyOf(allowedAccounts);
208205
}
209-
210-
public @Nonnull Long getLastModifiedTime() {
211-
return lastModifiedTime;
212-
}
213-
214-
public void setLastModifiedTime(@Nonnull Long lastModifiedTime) {
215-
this.lastModifiedTime = lastModifiedTime;
216-
}
217206
}
218207
}

‎orca/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/CompoundExecutionOperator.java‎

Lines changed: 54 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -19,26 +19,25 @@
1919
import com.netflix.spinnaker.kork.core.RetrySupport;
2020
import com.netflix.spinnaker.orca.api.pipeline.models.ExecutionType;
2121
import com.netflix.spinnaker.orca.api.pipeline.models.PipelineExecution;
22+
import com.netflix.spinnaker.orca.api.pipeline.models.StageExecution;
2223
import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository;
2324
import com.netflix.spinnaker.security.AuthenticatedRequest;
2425
import java.time.Duration;
2526
import java.util.function.Consumer;
27+
import java.util.function.Supplier;
2628
import javax.annotation.Nullable;
2729
import lombok.NonNull;
30+
import lombok.Value;
31+
import lombok.experimental.NonFinal;
2832
import lombok.extern.slf4j.Slf4j;
2933

3034
@Slf4j
35+
@Value
36+
@NonFinal
3137
public class CompoundExecutionOperator {
32-
private ExecutionRepository repository;
33-
private ExecutionRunner runner;
34-
private RetrySupport retrySupport;
35-
36-
public CompoundExecutionOperator(
37-
ExecutionRepository repository, ExecutionRunner runner, RetrySupport retrySupport) {
38-
this.repository = repository;
39-
this.runner = runner;
40-
this.retrySupport = retrySupport;
41-
}
38+
ExecutionRepository repository;
39+
ExecutionRunner runner;
40+
RetrySupport retrySupport;
4241

4342
public void cancel(ExecutionType executionType, String executionId) {
4443
cancel(
@@ -66,7 +65,7 @@ public void pause(
6665
@NonNull String executionId,
6766
@Nullable String pausedBy) {
6867
doInternal(
69-
(PipelineExecution execution) -> runner.reschedule(execution),
68+
runner::reschedule,
7069
() -> repository.pause(executionType, executionId, pausedBy),
7170
"pause",
7271
executionType,
@@ -79,33 +78,58 @@ public void resume(
7978
@Nullable String user,
8079
@NonNull Boolean ignoreCurrentStatus) {
8180
doInternal(
82-
(PipelineExecution execution) -> runner.unpause(execution),
81+
runner::unpause,
8382
() -> repository.resume(executionType, executionId, user, ignoreCurrentStatus),
8483
"resume",
8584
executionType,
8685
executionId);
8786
}
8887

89-
private void doInternal(
88+
public PipelineExecution updateStage(
89+
@NonNull ExecutionType executionType,
90+
@NonNull String executionId,
91+
@NonNull String stageId,
92+
@NonNull Consumer<StageExecution> stageUpdater) {
93+
return doInternal(
94+
runner::reschedule,
95+
() -> {
96+
PipelineExecution execution = repository.retrieve(executionType, executionId);
97+
StageExecution stage = execution.stageById(stageId);
98+
99+
// mutates stage in place
100+
stageUpdater.accept(stage);
101+
102+
repository.storeStage(stage);
103+
},
104+
"reschedule",
105+
executionType,
106+
executionId);
107+
}
108+
109+
private PipelineExecution doInternal(
90110
Consumer<PipelineExecution> runnerAction,
91111
Runnable repositoryAction,
92112
String action,
93113
ExecutionType executionType,
94114
String executionId) {
115+
PipelineExecution toReturn = null;
95116
try {
96-
runWithRetries(
97-
() -> {
98-
PipelineExecution execution = repository.retrieve(executionType, executionId);
99-
if (repository.handlesPartition(execution.getPartition())) {
100-
runnerAction.accept(execution);
101-
} else {
102-
log.info(
103-
"Not pushing queue message action='{}' for execution with foreign partition='{}'",
104-
action,
105-
execution.getPartition());
106-
}
107-
});
108117
runWithRetries(repositoryAction);
118+
119+
toReturn =
120+
runWithRetries(
121+
() -> {
122+
PipelineExecution execution = repository.retrieve(executionType, executionId);
123+
if (repository.handlesPartition(execution.getPartition())) {
124+
runnerAction.accept(execution);
125+
} else {
126+
log.info(
127+
"Not pushing queue message action='{}' for execution with foreign partition='{}'",
128+
action,
129+
execution.getPartition());
130+
}
131+
return execution;
132+
});
109133
} catch (Exception e) {
110134
log.error(
111135
"Failed to {} execution with executionType={} and executionId={}",
@@ -114,6 +138,11 @@ private void doInternal(
114138
executionId,
115139
e);
116140
}
141+
return toReturn;
142+
}
143+
144+
private <T> T runWithRetries(Supplier<T> action) {
145+
return retrySupport.retry(action, 5, Duration.ofMillis(100), false);
117146
}
118147

119148
private void runWithRetries(Runnable action) {

‎orca/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/StageExecutionImpl.java‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,7 @@
2020
import static java.lang.String.format;
2121
import static java.lang.String.join;
2222
import static java.nio.charset.StandardCharsets.UTF_8;
23-
import static java.util.Collections.emptyList;
24-
import static java.util.Collections.emptyMap;
25-
import static java.util.Collections.emptySet;
26-
import static java.util.Collections.singletonList;
23+
import static java.util.Collections.*;
2724
import static java.util.stream.Collectors.toList;
2825

2926
import com.fasterxml.jackson.annotation.JsonBackReference;
@@ -47,22 +44,14 @@
4744
import de.huxhorn.sulky.ulid.ULID;
4845
import java.io.IOException;
4946
import java.io.Serializable;
50-
import java.util.ArrayList;
51-
import java.util.Base64;
52-
import java.util.Collection;
53-
import java.util.HashMap;
54-
import java.util.HashSet;
55-
import java.util.LinkedList;
56-
import java.util.List;
57-
import java.util.Map;
58-
import java.util.Objects;
59-
import java.util.Optional;
60-
import java.util.Set;
47+
import java.util.*;
6148
import java.util.function.Predicate;
6249
import java.util.stream.Stream;
6350
import javax.annotation.Nonnull;
6451
import javax.annotation.Nullable;
52+
import lombok.extern.slf4j.Slf4j;
6553

54+
@Slf4j
6655
@JsonIdentityInfo(generator = ObjectIdGenerators.PropertyGenerator.class, property = "id")
6756
public class StageExecutionImpl implements StageExecution, Serializable {
6857

@@ -335,11 +324,22 @@ public void setScheduledTime(@Nullable Long scheduledTime) {
335324

336325
private LastModifiedDetails lastModified;
337326

338-
public @Nullable LastModifiedDetails getLastModified() {
327+
@Nullable
328+
@Override
329+
public StageExecution.LastModifiedDetails getLastModified() {
339330
return lastModified;
340331
}
341332

342333
public void setLastModified(@Nullable LastModifiedDetails lastModified) {
334+
if (lastModified != null
335+
&& this.lastModified != null
336+
&& lastModified.getLastModifiedTime() < this.lastModified.getLastModifiedTime()) {
337+
log.warn(
338+
"Setting lastModified to a value with an older timestamp, current={}, new={}",
339+
this.lastModified,
340+
lastModified);
341+
}
342+
343343
this.lastModified = lastModified;
344344
}
345345

‎orca/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/CompoundExecutionOperatorSpec.groovy‎

Lines changed: 74 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,24 @@
1717
package com.netflix.spinnaker.orca.pipeline
1818

1919
import com.netflix.spinnaker.kork.core.RetrySupport
20-
import com.netflix.spinnaker.orca.api.pipeline.models.ExecutionType
2120
import com.netflix.spinnaker.orca.api.pipeline.models.PipelineExecution
21+
import com.netflix.spinnaker.orca.api.pipeline.models.StageExecution
22+
import com.netflix.spinnaker.orca.pipeline.model.StageExecutionImpl
2223
import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository
2324
import org.slf4j.MDC
2425
import spock.lang.Specification
2526
import spock.lang.Subject
27+
import spock.lang.Unroll
2628

29+
import java.util.function.Consumer
30+
31+
import static com.netflix.spinnaker.orca.api.pipeline.models.ExecutionType.PIPELINE
2732

2833
class CompoundExecutionOperatorSpec extends Specification {
2934
ExecutionRepository repository = Mock(ExecutionRepository)
3035
ExecutionRunner runner = Mock(ExecutionRunner)
3136
def execution = Mock(PipelineExecution)
37+
def stage = Mock(StageExecution)
3238

3339
def setupSpec() {
3440
MDC.clear()
@@ -37,27 +43,83 @@ class CompoundExecutionOperatorSpec extends Specification {
3743
@Subject
3844
CompoundExecutionOperator operator = new CompoundExecutionOperator(repository, runner, new RetrySupport())
3945

40-
def 'should not push messages on the queue for foreign executions'() {
46+
@Unroll
47+
def '#method call should not push messages on the queue for foreign executions'() {
4148
when:
42-
operator.cancel(ExecutionType.PIPELINE, "id")
49+
operator."$method"(*args)
4350

44-
then: 'we never call runner.cancel(), only repository.cancel()'
45-
1 * repository.retrieve(ExecutionType.PIPELINE, "id") >> execution
51+
then: 'we never call runner.$method(), only repository.$method()'
4652
_ * execution.getPartition() >> "foreign"
53+
_ * execution.stageById('stageId') >> stage
54+
_ * stage.getId() >> "stageId"
55+
56+
_ * repository.retrieve(PIPELINE, "id") >> execution
4757
1 * repository.handlesPartition("foreign") >> false
48-
1 * repository.cancel(ExecutionType.PIPELINE, "id", "anonymous", null)
49-
0 * _
58+
1 * repository."$repoMethod"(*_)
59+
0 * runner._(*_)
60+
61+
where:
62+
method | repoMethod | args
63+
'cancel' | 'cancel' | [PIPELINE, "id"]
64+
'pause' | 'pause' | [PIPELINE, "id", "user"]
65+
'resume' | 'resume' | [PIPELINE, "id", "user", false]
66+
'updateStage' | 'storeStage' | [PIPELINE, "id", "stageId", {} ]
67+
}
68+
69+
@Unroll
70+
def 'should not push messages on the queue if the repository action fails'() {
71+
given:
72+
def runnerAction = Mock(Consumer)
73+
74+
when:
75+
def returnedExecution = operator.doInternal(
76+
runnerAction,
77+
{ throw new RuntimeException("repository action is failing") },
78+
"faily action",
79+
PIPELINE,
80+
"id"
81+
)
82+
83+
then: 'we never call the runner action'
84+
_ * repository.retrieve(PIPELINE, "id") >> execution
85+
_ * execution.getPartition() >> "local"
86+
_ * repository.handlesPartition("local") >> true
87+
88+
returnedExecution == null
89+
0 * runnerAction.accept(execution)
5090
}
5191

52-
def 'should handle both the queue and the execution repository for local executions'() {
92+
@Unroll
93+
def '#method call should handle both the queue and the execution repository for local executions'() {
5394
when:
54-
operator.cancel(ExecutionType.PIPELINE, "id")
95+
operator."$method"(*args)
5596

5697
then: 'we call both runner.cancel() and repository.cancel()'
57-
1 * repository.retrieve(ExecutionType.PIPELINE, "id") >> execution
98+
1 * repository.retrieve(PIPELINE, "id") >> execution
5899
_ * execution.getPartition() >> "local"
59100
1 * repository.handlesPartition("local") >> true
60-
1 * repository.cancel(ExecutionType.PIPELINE, "id", "anonymous", null)
61-
1 * runner.cancel(execution, "anonymous", null)
101+
1 * repository."$method"(*_)
102+
1 * runner."$runnerMethod"(*_)
103+
104+
where:
105+
method | runnerMethod | args
106+
'cancel' | 'cancel' | [PIPELINE, "id"]
107+
'pause' | 'reschedule' | [PIPELINE, "id", "user"]
108+
'resume' | 'unpause' | [PIPELINE, "id", "user", false]
109+
}
110+
111+
def 'updateStage() updates the stage'() {
112+
given:
113+
StageExecutionImpl stage = new StageExecutionImpl(id: 'stageId')
114+
115+
when:
116+
operator.updateStage(PIPELINE, 'id', 'stageId',
117+
{ it.setLastModified(new StageExecution.LastModifiedDetails(user: 'user')) })
118+
119+
then:
120+
_ * repository.retrieve(PIPELINE, 'id') >> execution
121+
1 * execution.stageById('stageId') >> stage
122+
1 * repository.storeStage(stage)
123+
stage.getLastModified().getUser() == 'user'
62124
}
63125
}

‎orca/orca-interlink/src/main/java/com/netflix/spinnaker/orca/interlink/aws/InterlinkAmazonMessageHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ void handleInternal(InterlinkEvent event) {
6565
MDC.put(Header.EXECUTION_ID.getHeader(), event.getExecutionId());
6666

6767
if (executionRepository.handlesPartition(event.getPartition())) {
68-
event.applyTo(executionOperator);
68+
event.withObjectMapper(objectMapper).applyTo(executionOperator);
6969
} else {
7070
log.debug(
7171
"Execution repository with local partition {} can't handle this event {} so it will not be applied",

0 commit comments

Comments
 (0)