Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update jpmml project dependencies #66

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ lazy val root = project
.enablePlugins(ScalaUnidocPlugin)
.settings(
name := "flink-jpmml",
crossScalaVersions := Seq("2.10.6", "2.11.11"),
crossScalaVersions := Seq("2.11.12"),
unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(`flink-jpmml-examples`,
`flink-jpmml-assets`)
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ import io.radicalbit.flink.pmml.scala.api.exceptions.{InputValidationException,
import io.radicalbit.flink.pmml.scala.api.pipeline.Pipeline
import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
import io.radicalbit.flink.pmml.scala.models.prediction.Prediction
import org.apache.commons.io.input.ReaderInputStream
import org.apache.flink.ml.math.Vector
import org.dmg.pmml.FieldName
import org.jpmml.evaluator._
import org.jpmml.model.{ImportFilter, JAXBUtil}
import org.xml.sax.InputSource
import org.jpmml.model.{PMMLUtil => MarshalUtil}

import scala.collection.JavaConversions._
import scala.util.Try
Expand All @@ -54,11 +54,11 @@ object PmmlModel {
val readerFromFs = reader.buildDistributedPath
val result = fromFilteredSource(readerFromFs)

new PmmlModel(Evaluator(evaluatorInstance.newModelEvaluator(JAXBUtil.unmarshalPMML(result))))
new PmmlModel(Evaluator(evaluatorInstance.newModelEvaluator(result)))
}

private def fromFilteredSource(PMMLPath: String) =
JAXBUtil.createFilteredSource(new InputSource(new StringReader(PMMLPath)), new ImportFilter())
MarshalUtil.unmarshal(new ReaderInputStream(new StringReader(PMMLPath)))

/** It provides a new instance of the [[PmmlModel]] with [[EmptyEvaluator]]
*
Expand Down Expand Up @@ -93,7 +93,7 @@ class PmmlModel(private[api] val evaluator: Evaluator) extends Pipeline {
*
* - `validateInput` validates the input to be conform to PMML model size
*
* - `prepareInput` prepares the input in full compliance to [[org.jpmml.evaluator.EvaluatorUtil.prepare]] JPMML method
* - `prepareInput` prepares the input in full compliance to [[InputField]] preparation JPMML method
*
* - `evaluateInput` evaluates the input against inner PMML model instance and returns a Java Map output
*
Expand Down Expand Up @@ -134,7 +134,7 @@ class PmmlModel(private[api] val evaluator: Evaluator) extends Pipeline {
}

/** Binds each field with input value and prepare the record to be evaluated
* by [[EvaluatorUtil.prepare]] method.
* by [[InputField]] preparation method.
*
* @param input Validated input as a [[Map]] keyed by field name
* @param replaceNaN Optional replace value in case of missing values
Expand All @@ -146,7 +146,7 @@ class PmmlModel(private[api] val evaluator: Evaluator) extends Pipeline {

activeFields.map { field =>
val rawValue = input.get(field.getName.getValue).orElse(replaceNaN).orNull
prepareAndEmit(Try { EvaluatorUtil.prepare(field, rawValue) }, field.getName)
prepareAndEmit(Try { field.prepare(rawValue) }, field.getName)
}.toMap

}
Expand All @@ -164,7 +164,7 @@ class PmmlModel(private[api] val evaluator: Evaluator) extends Pipeline {
* @param evaluationResult outcome from JPMML evaluation
* @return The prediction value as a [Double]
*/
private[api] def extractTarget(evaluationResult: java.util.Map[FieldName, _]): Double = {
private[api] def extractTarget(evaluationResult: util.Map[FieldName, _]): Double = {
val targets = extractTargetFields(evaluationResult)

targets.headOption.flatMap {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ class PmmlModelSpec extends WordSpec with Matchers with PmmlLoaderKit with PmmlE
"Pipeline.prepareAndEmit" should {

"emit valid prepared value if the input is correct" in {
val v = FieldValueUtil.create(null.asInstanceOf[DataType], null.asInstanceOf[OpType], 1.0)
val v = FieldValueUtil.create(1.0)
val d = createDataField()
model.prepareAndEmit(Success(v), d.getName) shouldBe (d.getName, v)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class PredictionSpec extends WordSpec with Matchers {
}

"return None if onFailedPrediction is active and EvaluationException" in {
throwableFunc(new EvaluationException) {
throwableFunc(new EvaluationException("")) {
case e: Throwable => Prediction.onFailedPrediction(e)
} shouldBe Prediction(EmptyScore)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ trait PmmlEvaluatorKit {

final protected def buildExpectedPreparedMap(in: Map[String, Any], keys: Seq[String], replaceValue: Option[Double]) =
keys.map {
case k if in.contains(k) => new FieldName(k) -> FieldValueUtil.create(null, null, in(k))
case emptyKey => new FieldName(emptyKey) -> FieldValueUtil.create(null, null, replaceValue.orNull)
case k if in.contains(k) => new FieldName(k) -> FieldValueUtil.create(in(k))
case emptyKey => new FieldName(emptyKey) -> FieldValueUtil.create(replaceValue.orNull)
} toMap

}
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@
package io.radicalbit.flink.pmml.scala.utils

import org.dmg.pmml.PMML
import org.jpmml.model.{ImportFilter, JAXBUtil}
import org.xml.sax.InputSource
import org.jpmml.model.PMMLUtil

trait PmmlLoaderKit {

Expand All @@ -42,9 +41,6 @@ trait PmmlLoaderKit {
final protected def getPMMLSource(path: String): String =
getClass.getResource(path).getPath

final protected def getPMMLResource(path: String): PMML = {
val source = scala.io.Source.fromURL(getClass.getResource(path)).reader()
JAXBUtil.unmarshalPMML(ImportFilter.apply(new InputSource(source)))
}
final protected def getPMMLResource(path: String): PMML = PMMLUtil.unmarshal(getClass.getResourceAsStream(path))

}
4 changes: 2 additions & 2 deletions project/Dependencies.scala
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ object Dependencies {

private object jpmml {
lazy val namespace = "org.jpmml"
lazy val version = "1.3.9"
lazy val version = "1.4.7"
lazy val evaluator = namespace % "pmml-evaluator" % version
}

Expand All @@ -84,7 +84,7 @@ object Dependencies {

private object hadoop {
lazy val namespace = "org.apache.hadoop"
lazy val version = "2.3.0"
lazy val version = "2.8.3"
lazy val hdfs = namespace % "hadoop-hdfs" % version
lazy val common = namespace % "hadoop-common" % version
lazy val mincluster = namespace % "hadoop-minicluster" % version
Expand Down
2 changes: 1 addition & 1 deletion version.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@
*
*/

version in ThisBuild := "0.7.0-SNAPSHOT"
version in ThisBuild := "0.7.1-SNAPSHOT"