Skip to content

Commit

Permalink
Add capability matcher plugin type
Browse files Browse the repository at this point in the history
  • Loading branch information
michel-kraemer committed Oct 22, 2024
1 parent e70e4fc commit ca85be6
Show file tree
Hide file tree
Showing 11 changed files with 194 additions and 13 deletions.
6 changes: 3 additions & 3 deletions src/main/kotlin/agent/RemoteAgentRegistry.kt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import AddressConstants.REMOTE_AGENT_ADDED
import AddressConstants.REMOTE_AGENT_ADDRESS_PREFIX
import AddressConstants.REMOTE_AGENT_LEFT
import agent.AgentRegistry.SelectCandidatesParam
import helper.CapabilitiesMatcher
import helper.CapabilityMatcher
import helper.JsonUtils
import helper.debounce
import helper.hazelcast.ClusterMap
Expand Down Expand Up @@ -116,7 +116,7 @@ class RemoteAgentRegistry(private val vertx: Vertx) : AgentRegistry, CoroutineSc
* Matches required capabilities of process chains against provided
* capabilities of agents
*/
private val capabilitiesMatcher = CapabilitiesMatcher()
private val capabilityMatcher = CapabilityMatcher()

init {
// create shared maps
Expand Down Expand Up @@ -281,7 +281,7 @@ class RemoteAgentRegistry(private val vertx: Vertx) : AgentRegistry, CoroutineSc
// agent's lastSequence would definitely be higher
continue
}
if (capabilitiesMatcher.matches(ps.requiredCapabilities, supportedCapabilities)) {
if (capabilityMatcher.matches(ps.requiredCapabilities, supportedCapabilities)) {
// the agent supports at least one required capabilities set
shouldInquire = true
break
Expand Down
14 changes: 7 additions & 7 deletions src/main/kotlin/cloud/SetupSelector.kt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package cloud

import db.VMRegistry
import helper.CapabilitiesMatcher
import helper.CapabilityMatcher
import model.cloud.PoolAgentParams
import model.setup.Setup
import org.slf4j.LoggerFactory
Expand All @@ -22,7 +22,7 @@ class SetupSelector(private val vmRegistry: VMRegistry,
/**
* Matches required capabilities with those provided by setups
*/
private val capabilitiesMatcher = CapabilitiesMatcher()
private val capabilityMatcher = CapabilityMatcher()

/**
* Iterate through [nVMsPerSetup] and [nCreatedPerSetup] and count how many
Expand All @@ -35,7 +35,7 @@ class SetupSelector(private val vmRegistry: VMRegistry,
.groupBy { it.key }.mapValues { e -> e.value.sumOf { v -> v.value } }

return merged.map { (setup, n) ->
if (capabilitiesMatcher.matches(capabilities, setup.providedCapabilities)) {
if (capabilityMatcher.matches(capabilities, setup.providedCapabilities)) {
n
} else {
0
Expand Down Expand Up @@ -68,7 +68,7 @@ class SetupSelector(private val vmRegistry: VMRegistry,
setups: List<Setup>): List<Setup> {
// select candidate setups that satisfy the given required capabilities
val matchingSetups = setups.filter { s ->
capabilitiesMatcher.matches(requiredCapabilities, s.providedCapabilities)
capabilityMatcher.matches(requiredCapabilities, s.providedCapabilities)
}
if (matchingSetups.isEmpty()) {
return emptyList()
Expand Down Expand Up @@ -99,7 +99,7 @@ class SetupSelector(private val vmRegistry: VMRegistry,

// check if we already have enough VMs that provide similar capabilities
for (params in poolAgentParams) {
if (params.max != null && capabilitiesMatcher.matches(params.capabilities,
if (params.max != null && capabilityMatcher.matches(params.capabilities,
setup.providedCapabilities)) {
// Creating a new VM with [setup] would add an agent with the given
// provided capabilities. Check if this would exceed the maximum
Expand Down Expand Up @@ -165,7 +165,7 @@ class SetupSelector(private val vmRegistry: VMRegistry,
val papi = pap.iterator()
while (papi.hasNext()) {
val p = papi.next()
if (capabilitiesMatcher.matches(p.capabilities, setup.providedCapabilities)) {
if (capabilityMatcher.matches(p.capabilities, setup.providedCapabilities)) {
// we found parameters our setup satisfies
if (p.min > minimum) {
minimum = min(p.min, maximum)
Expand Down Expand Up @@ -195,7 +195,7 @@ class SetupSelector(private val vmRegistry: VMRegistry,
val i = result.iterator()
while (i.hasNext()) {
val setup = i.next()
if (capabilitiesMatcher.matches(p.capabilities, setup.providedCapabilities)) {
if (capabilityMatcher.matches(p.capabilities, setup.providedCapabilities)) {
if (count == p.max) {
i.remove()
} else {
Expand Down
8 changes: 8 additions & 0 deletions src/main/kotlin/db/PluginRegistry.kt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package db

import model.plugins.CapabilityMatcherPlugin
import model.plugins.InitializerPlugin
import model.plugins.InputAdapterPlugin
import model.plugins.OutputAdapterPlugin
Expand All @@ -15,6 +16,8 @@ import model.plugins.SetupAdapterPlugin
* @author Michel Kraemer
*/
class PluginRegistry(private val compiledPlugins: List<Plugin>) {
private val capabilityMatchers = compiledPlugins
.filterIsInstance<CapabilityMatcherPlugin>()
private val initializers = compiledPlugins.filterIsInstance<InitializerPlugin>()
.toResolved()
private val inputAdapters = compiledPlugins.filterIsInstance<InputAdapterPlugin>()
Expand All @@ -40,6 +43,11 @@ class PluginRegistry(private val compiledPlugins: List<Plugin>) {
*/
fun getAllPlugins() = compiledPlugins

/**
* Get all capability matchers
*/
fun getCapabilityMatchers() = capabilityMatchers

/**
* Get all initializers
*/
Expand Down
4 changes: 4 additions & 0 deletions src/main/kotlin/db/PluginRegistryFactory.kt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import io.vertx.core.Vertx
import io.vertx.core.json.JsonArray
import io.vertx.core.json.JsonObject
import io.vertx.kotlin.coroutines.coAwait
import model.plugins.CapabilityMatcherPlugin
import model.plugins.InitializerPlugin
import model.plugins.InputAdapterPlugin
import model.plugins.OutputAdapterPlugin
Expand All @@ -16,6 +17,7 @@ import model.plugins.ProcessChainConsistencyCheckerPlugin
import model.plugins.ProgressEstimatorPlugin
import model.plugins.RuntimePlugin
import model.plugins.SetupAdapterPlugin
import model.plugins.capabilityMatcherPluginTemplate
import model.plugins.initializerPluginTemplate
import model.plugins.inputAdapterPluginTemplate
import model.plugins.outputAdapterPluginTemplate
Expand Down Expand Up @@ -237,6 +239,8 @@ object PluginRegistryFactory {

@Suppress("UNCHECKED_CAST")
return when (plugin) {
is CapabilityMatcherPlugin -> plugin.copy(compiledFunction = wrapPluginFunction(
f as KFunction<Int>, ::capabilityMatcherPluginTemplate.parameters))
is InitializerPlugin -> plugin.copy(compiledFunction = wrapPluginFunction(
f as KFunction<Unit>, ::initializerPluginTemplate.parameters))
is InputAdapterPlugin -> plugin.copy(compiledFunction = wrapPluginFunction(
Expand Down
2 changes: 1 addition & 1 deletion src/main/kotlin/helper/CapabilitiesMatcher.kt
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ package helper
* [CapabilityMatcherPlugin]s doing custom comparisons.
* @author Michel Kraemer
*/
class CapabilitiesMatcher {
class CapabilityMatcher {
/**
* Matches a collection of [requiredCapabilities] with a collection of
* [providedCapabilities]. Returns `true` if the collection of provided
Expand Down
108 changes: 108 additions & 0 deletions src/main/kotlin/model/plugins/CapabilitiesMatcherPlugin.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package model.plugins

import com.fasterxml.jackson.annotation.JsonIgnore
import io.vertx.core.Vertx
import kotlin.reflect.KFunction
import kotlin.reflect.full.callSuspend

/**
* Parameters that will be passed to a [CapabilityMatcherPlugin]
*/
interface CapabilityMatcherParams {
/**
* The required capability that should be matched with the collection of
* [providedCapabilities]. In the simplest case, the plugin may just check
* if the collection contains this string, but more complex comparisons
* can be implemented.
*/
val subject: String

/**
* The collection of provided capabilities to which the required capability
* denoted with [subject] should be matched.
*/
val providedCapabilities: Collection<String>

/**
* The collection of all required capabilities (including [subject]) for
* reference or if complex decisions need to be made (e.g. [subject] `"A"`
* only matches [providedCapabilities] if [allRequiredCapabilities] does not
* contain string `"B"`).
*/
val allRequiredCapabilities: Collection<String>
}

/**
* A simple implementation of [CapabilityMatcherParams]
*/
data class SimpleCapabilityMatcherParams(
override val subject: String,
override val providedCapabilities: Collection<String>,
override val allRequiredCapabilities: Collection<String>
) : CapabilityMatcherParams

/**
* A capability matcher plugin is a function that can change how Steep decides
* whether a given collection of provided capabilities satisfy a required
* capability. The function has the following signature:
*
* suspend fun myCapabilityMatcher(
* params: model.plugins.CapabilityMatcherParams,
* vertx: io.vertx.core.Vertx): Int
*
* The function will be called with a set of [CapabilityMatcherParams] and
* the Vert.x instance. If required, the function can be a suspend function.
*
* The parameters contain a [CapabilityMatcherParams.subject] (representing
* a single required capability) and a collection of
* [CapabilityMatcherParams.providedCapabilities].
*
* For reference or if more complex decisions need to be made, the parameters
* also contain the collection of all required capabilities (including the
* subject).
*
* The function can cast a vote on whether the provided capabilities satisfy
* the required capability denoted by [CapabilityMatcherParams.subject]. For
* this, it returns an integer value. Positive values mean that the provided
* capabilities match the subject. The higher the value, the more certain the
* function is about this. A value of [Int.MAX_VALUE] means that the
* capabilities definitely match (i.e. the plugin is absolutely certain). No
* other plugin will be called in this case.
*
* Negative values mean that the provided capabilities *do not* match the
* subject. The lower the value, the more certain the function is about this.
* A value of [Int.MIN_VALUE] means that the plugin is absolutely certain that
* capabilities *do not* match. No other plugin will be called in this case.
*
* A value of 0 (zero) means that the plugin is unable to tell if the provided
* capabilities match the subject. The decision should be made based on other
* criteria (i.e. by calling other plugins or by simply comparing strings as
* a fallback).
*/
data class CapabilityMatcherPlugin(
override val name: String,
override val scriptFile: String,
override val version: String? = null,

/**
* The compiled plugin
*/
@JsonIgnore
override val compiledFunction: KFunction<Int> = throwPluginNeedsCompile()
) : Plugin

@Suppress("UNUSED_PARAMETER")
internal fun capabilityMatcherPluginTemplate(params: CapabilityMatcherParams,
vertx: Vertx): Int {
throw NotImplementedError("This is just a template specifying the " +
"function signature of a setup adapter plugin")
}

suspend fun CapabilityMatcherPlugin.call(params: CapabilityMatcherParams,
vertx: Vertx): Int {
return if (this.compiledFunction.isSuspend) {
this.compiledFunction.callSuspend(params, vertx)
} else {
this.compiledFunction.call(params, vertx)
}
}
1 change: 1 addition & 0 deletions src/main/kotlin/model/plugins/Plugin.kt
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import kotlin.reflect.jvm.javaType
include = JsonTypeInfo.As.PROPERTY,
property = "type")
@JsonSubTypes(
JsonSubTypes.Type(value = CapabilityMatcherPlugin::class, name = "capabilityMatcher"),
JsonSubTypes.Type(value = InitializerPlugin::class, name = "initializer"),
JsonSubTypes.Type(value = InputAdapterPlugin::class, name = "inputAdapter"),
JsonSubTypes.Type(value = OutputAdapterPlugin::class, name = "outputAdapter"),
Expand Down
4 changes: 2 additions & 2 deletions src/main/kotlin/model/plugins/SetupAdapterPlugin.kt
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ import kotlin.reflect.full.callSuspend
*
* suspend fun mySetupAdapter(setup: model.setup.Setup,
* requiredCapabilities: Collection<String>,
* vertx: io.vertx.core.Vertx): model.setup.Setup,
* vertx: io.vertx.core.Vertx): model.setup.Setup
*
* The function will be called with a setup to modify and a collection of
* required capabilities the modified setup should meet. The function should
* return a new setup instance or the original one if no modifications were
* necessary.
* necessary. If required, the function can be a suspend function.
*/
data class SetupAdapterPlugin(
override val name: String,
Expand Down
46 changes: 46 additions & 0 deletions src/test/kotlin/db/PluginRegistryTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@ import io.vertx.kotlin.core.json.jsonObjectOf
import io.vertx.kotlin.coroutines.dispatcher
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.launch
import model.plugins.CapabilityMatcherPlugin
import model.plugins.InitializerPlugin
import model.plugins.InputAdapterPlugin
import model.plugins.OutputAdapterPlugin
import model.plugins.ProcessChainAdapterPlugin
import model.plugins.ProgressEstimatorPlugin
import model.plugins.RuntimePlugin
import model.plugins.SetupAdapterPlugin
import model.plugins.SimpleCapabilityMatcherParams
import model.plugins.call
import model.processchain.Argument
import model.processchain.ArgumentVariable
Expand Down Expand Up @@ -86,6 +88,50 @@ class PluginRegistryTest {
}
}

/**
* Test if [PluginRegistry.getCapabilityMatchers] works correctly
*/
@Test
fun getCapabilityMatchers() {
val cm1 = CapabilityMatcherPlugin("a", "file.kts")
val cm2 = CapabilityMatcherPlugin("b", "file2.kts")
val cm3 = CapabilityMatcherPlugin("c", "file3.kts")
val expected = listOf(cm1, cm2, cm3)
val pr = PluginRegistry(expected)
assertThat(pr.getCapabilityMatchers()).isEqualTo(expected)
assertThat(pr.getCapabilityMatchers()).isNotSameAs(expected)
}

/**
* Test if a simple capability matcher can be compiled and executed
*/
@Test
fun compileDummyCapabilityMatcher(vertx: Vertx, ctx: VertxTestContext) {
CoroutineScope(vertx.dispatcher()).launch {
val config = jsonObjectOf(
ConfigConstants.PLUGINS to "src/**/db/dummyCapabilityMatcher.yaml"
)
PluginRegistryFactory.initialize(vertx, config)

val pr = PluginRegistryFactory.create()
val matchers = pr.getCapabilityMatchers()
ctx.coVerify {
assertThat(matchers).hasSize(1)
val matcher = matchers[0]

val matches = matcher.call(SimpleCapabilityMatcherParams(
"elvis", listOf("max", "elvis", "sean"), listOf("elvis")), vertx)
assertThat(matches).isEqualTo(1)

val doesNotMatch = matcher.call(SimpleCapabilityMatcherParams(
"elvis", listOf("max", "sean"), listOf("elvis")), vertx)
assertThat(doesNotMatch).isEqualTo(-1)
}

ctx.completeNow()
}
}

/**
* Test if [PluginRegistry.getInitializers] works correctly
*/
Expand Down
9 changes: 9 additions & 0 deletions src/test/resources/db/dummyCapabilityMatcher.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import model.plugins.CapabilityMatcherParams

fun dummyCapabilityMatcher(params: CapabilityMatcherParams): Int {
return if (params.providedCapabilities.contains(params.subject)) {
1
} else {
-1
}
}
5 changes: 5 additions & 0 deletions src/test/resources/db/dummyCapabilityMatcher.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# A capability matcher that returns 1 if `subject` is contained in the
# collection of provided capabilities and -1 if it's not.
- name: dummyCapabilityMatcher
type: capabilityMatcher
scriptFile: db/dummyCapabilityMatcher.kts

0 comments on commit ca85be6

Please sign in to comment.