From e2a3499fc1e3530487a6bc8b69e1595710168984 Mon Sep 17 00:00:00 2001 From: "GARCIA, JOSE" Date: Thu, 24 Oct 2024 16:26:01 -0700 Subject: [PATCH] feat(pipeline/configuration): Updated pipeline level configuration that introduces a new property named metadata. SpEL processing pipeline.metadata so that it is resolved/ready when referenced. --- .../pipeline/models/PipelineExecution.java | 4 ++ .../orca/pipeline/EvaluateVariablesStage.java | 15 ++++++ .../orca/pipeline/ExecutionLauncher.java | 1 + .../orca/pipeline/model/PipelineBuilder.java | 5 ++ .../pipeline/model/PipelineExecutionImpl.java | 10 ++++ .../tasks/EvaluateVariablesStageSpec.groovy | 21 ++++++++ .../pipeline/model/PipelineBuilderTest.java | 12 +++++ .../model/PipelineExecutionImplTest.java | 10 ++++ .../EchoNotifyingExecutionListener.groovy | 5 ++ ...ifyingPipelineExecutionListenerSpec.groovy | 50 +++++++++++++++++++ 10 files changed, 133 insertions(+) diff --git a/orca-api/src/main/java/com/netflix/spinnaker/orca/api/pipeline/models/PipelineExecution.java b/orca-api/src/main/java/com/netflix/spinnaker/orca/api/pipeline/models/PipelineExecution.java index d7ba723c1c..38187e4225 100644 --- a/orca-api/src/main/java/com/netflix/spinnaker/orca/api/pipeline/models/PipelineExecution.java +++ b/orca-api/src/main/java/com/netflix/spinnaker/orca/api/pipeline/models/PipelineExecution.java @@ -34,6 +34,10 @@ public interface PipelineExecution { @Nonnull ExecutionType getType(); + Map getMetadata(); + + void setMetadata(Map metadata); + String getId(); void setId(String id); diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/EvaluateVariablesStage.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/EvaluateVariablesStage.java index 1949417799..be375c8ed2 100644 --- a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/EvaluateVariablesStage.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/EvaluateVariablesStage.java @@ -24,6 +24,7 @@ import com.netflix.spinnaker.orca.api.pipeline.graph.TaskNode; import com.netflix.spinnaker.orca.api.pipeline.models.StageExecution; import com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator; +import com.netflix.spinnaker.orca.pipeline.model.PipelineExecutionImpl; import com.netflix.spinnaker.orca.pipeline.model.StageContext; import com.netflix.spinnaker.orca.pipeline.tasks.EvaluateVariablesTask; import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor; @@ -76,6 +77,20 @@ public boolean processExpressions( EvaluateVariablesStageContext context = stage.mapTo(EvaluateVariablesStageContext.class); StageContext augmentedContext = contextParameterProcessor.buildExecutionContext(stage); + + PipelineExecutionImpl pipelineExecution = + (PipelineExecutionImpl) augmentedContext.get("execution"); + // Evaluate spel expressions on pipeline.metadata so that they are resolved before evaluating at + // stage level. + // This is needed for in case pipeline.metadata is referenced at stage level. + Map evaluatedPipelineMetadata = + contextParameterProcessor.process( + mapper.convertValue( + pipelineExecution.getMetadata(), new TypeReference>() {}), + augmentedContext, + true); + pipelineExecution.setMetadata(evaluatedPipelineMetadata); + Map varSourceToEval = new HashMap<>(); int lastFailedCount = 0; diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java index 6255292179..ffec881a6c 100644 --- a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/ExecutionLauncher.java @@ -231,6 +231,7 @@ private PipelineExecution parsePipeline(Map config) { .withSpelEvaluator(getString(config, "spelEvaluator")) .withTemplateVariables(getMap(config, "templateVariables")) .withIncludeAllowedAccounts(executionConfigurationProperties.isIncludeAllowedAccounts()) + .withMetadata(getMap(config, "metadata")) .build(); } diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.java index c3c56c2a73..49e65ae70e 100644 --- a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilder.java @@ -32,6 +32,11 @@ public PipelineBuilder(String application) { pipeline = PipelineExecutionImpl.newPipeline(application); } + public PipelineBuilder withMetadata(Map metadata) { + pipeline.setMetadata(metadata); + return this; + } + public PipelineBuilder withIncludeAllowedAccounts(boolean includeAllowedAccounts) { this.includeAllowedAccounts = includeAllowedAccounts; return this; diff --git a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineExecutionImpl.java b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineExecutionImpl.java index 1748ab315e..bd589dbb34 100644 --- a/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineExecutionImpl.java +++ b/orca-core/src/main/java/com/netflix/spinnaker/orca/pipeline/model/PipelineExecutionImpl.java @@ -74,6 +74,16 @@ public PipelineExecutionImpl( return type; } + private Map metadata; + + public Map getMetadata() { + return this.metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + private String id; public @Nonnull String getId() { diff --git a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/tasks/EvaluateVariablesStageSpec.groovy b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/tasks/EvaluateVariablesStageSpec.groovy index 4a6219c51f..2515f00333 100644 --- a/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/tasks/EvaluateVariablesStageSpec.groovy +++ b/orca-core/src/test/groovy/com/netflix/spinnaker/orca/pipeline/tasks/EvaluateVariablesStageSpec.groovy @@ -82,6 +82,27 @@ class EvaluateVariablesStageSpec extends Specification { stage.context.notifications[0].address == "someone@somewhere.com" } + void "Should eval successful when referencing pipeline metadata"() { + setup: + def summary = new ExpressionEvaluationSummary() + + def stage = stage { + refId = "1" + type = "evaluateVariables" + context["notifications"] = [ + [address: '${execution.metadata.myaddress}'] + ] + execution["metadata"] = [myaddress: "someone@somewhere.com"] + } + + when: + def shouldContinue = evaluateVariablesStage.processExpressions(stage, contextParameterProcessor, summary) + + then: + shouldContinue == false + stage.context.notifications[0].address == "someone@somewhere.com" + } + void "Should correctly clean variables in restart scenario"() { setup: def summary = new ExpressionEvaluationSummary() diff --git a/orca-core/src/test/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilderTest.java b/orca-core/src/test/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilderTest.java index c2028aad33..4355f6f2c7 100644 --- a/orca-core/src/test/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilderTest.java +++ b/orca-core/src/test/java/com/netflix/spinnaker/orca/pipeline/model/PipelineBuilderTest.java @@ -81,4 +81,16 @@ void buildExcludesAllowedAccountsWhenFalse() { // then assertThat(execution.getAuthentication().getAllowedAccounts()).isEqualTo(Set.of()); } + + @Test + void buildInlcludesMetadata() { + + // when + PipelineBuilder pipelineBuilder = + new PipelineBuilder("my-application").withMetadata(new HashMap()); + PipelineExecution execution = pipelineBuilder.build(); + + // then + assertThat(execution.getMetadata()).isNotNull(); + } } diff --git a/orca-core/src/test/java/com/netflix/spinnaker/orca/pipeline/model/PipelineExecutionImplTest.java b/orca-core/src/test/java/com/netflix/spinnaker/orca/pipeline/model/PipelineExecutionImplTest.java index 629b043287..805e865182 100644 --- a/orca-core/src/test/java/com/netflix/spinnaker/orca/pipeline/model/PipelineExecutionImplTest.java +++ b/orca-core/src/test/java/com/netflix/spinnaker/orca/pipeline/model/PipelineExecutionImplTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import com.netflix.spinnaker.orca.api.pipeline.models.ExecutionType; +import java.util.HashMap; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -74,4 +75,13 @@ void getTotalSizeCompleteInfo() { // then assertThat(pipelineExecution.getTotalSize().get()).isEqualTo(pipelineSize + stageSize); } + + @Test + void getMetadata() { + // given + pipelineExecution.setMetadata(new HashMap()); + + // then + assertThat(pipelineExecution.getMetadata()).isNotNull(); + } } diff --git a/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingExecutionListener.groovy b/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingExecutionListener.groovy index 2da2b6c01a..11c58eebf4 100644 --- a/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingExecutionListener.groovy +++ b/orca-echo/src/main/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingExecutionListener.groovy @@ -104,6 +104,11 @@ class EchoNotifyingExecutionListener implements ExecutionListener { } private void processSpelInNotifications(PipelineExecution execution) { + //Evaluate spel expressions on pipeline.metadata so that they are resolved before evaluating notifications. + //This is needed so that the values are ready for when they are referenced at pipeline notification level. + Map evaluatedPipelineMetadata = contextParameterProcessor.process(objectMapper.convertValue(execution.getMetadata(), Map), contextParameterProcessor.buildExecutionContext(execution), true) + execution.setMetadata(evaluatedPipelineMetadata) + List> spelProcessedNotifications = execution.notifications.collect({ contextParameterProcessor.process(it, contextParameterProcessor.buildExecutionContext(execution), true) }) diff --git a/orca-echo/src/test/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingPipelineExecutionListenerSpec.groovy b/orca-echo/src/test/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingPipelineExecutionListenerSpec.groovy index 90eb9da359..e7960b2efb 100644 --- a/orca-echo/src/test/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingPipelineExecutionListenerSpec.groovy +++ b/orca-echo/src/test/groovy/com/netflix/spinnaker/orca/echo/spring/EchoNotifyingPipelineExecutionListenerSpec.groovy @@ -177,6 +177,56 @@ class EchoNotifyingPipelineExecutionListenerSpec extends Specification { 0 * _ } + void "evaluate SpEL that references pipeline.metadata using beforeExecution"(){ + given: + def pipelineMetadata = ['mychainId': '53dd5f3e-fb0d-4c48-92ee-fda7f0eca5e5'] + def pipeline = PipelineExecutionImpl.newPipeline("myapp") + pipeline.setMetadata(pipelineMetadata) + def pipelineConfiguredNotification = [ + when : ["pipeline.started", "pipeline.completed"], + type : "slack", + address: 'spinnaker', + customData: [chainId: "\${execution.metadata.mychainId}"] + ] + pipeline.notifications.add(pipelineConfiguredNotification) + + when: + echoListener.beforeExecution(null, pipeline) + + then: + pipeline.notifications.size() == 1 + pipeline.notifications[0].when.containsAll(["pipeline.started", "pipeline.completed"]) + pipeline.notifications[0].customData.chainId == pipelineMetadata.mychainId + 1 * front50Service.getApplicationNotifications("myapp") >> new ApplicationNotifications() + 1 * echoService.recordEvent(_) + 0 * _ + } + + void "evaluate SpEL that references pipeline.metadata using afterExecution"(){ + given: + def pipelineMetadata = ['mychainId': '53dd5f3e-fb0d-4c48-92ee-fda7f0eca5e5'] + def pipeline = PipelineExecutionImpl.newPipeline("myapp") + pipeline.setMetadata(pipelineMetadata) + def pipelineConfiguredNotification = [ + when : ["pipeline.started", "pipeline.completed"], + type : "slack", + address: 'spinnaker', + customData: [chainId: "\${execution.metadata.mychainId}"] + ] + pipeline.notifications.add(pipelineConfiguredNotification) + + when: + echoListener.afterExecution(null, pipeline, ExecutionStatus.SUCCEEDED, true) + + then: + pipeline.notifications.size() == 1 + pipeline.notifications[0].when.containsAll(["pipeline.started", "pipeline.completed"]) + pipeline.notifications[0].customData.chainId == pipelineMetadata.mychainId + 1 * front50Service.getApplicationNotifications("myapp") >> new ApplicationNotifications() + 1 * echoService.recordEvent(_) + 0 * _ + } + void "handles case where no notifications are present"() { given: def pipeline = PipelineExecutionImpl.newPipeline("myapp")