diff --git a/build.sbt b/build.sbt index 749bdde49..2528e6702 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ name := "codepropertygraph" // parsed by project/Versions.scala, updated by updateDependencies.sh -val overflowdbVersion = "1.64" +val overflowdbVersion = "1.68" inThisBuild( List( @@ -71,6 +71,7 @@ lazy val codepropertygraph = Projects.codepropertygraph lazy val semanticcpg = Projects.semanticcpg lazy val macros = Projects.macros lazy val schema2json = Projects.schema2json +lazy val performance = Projects.performance // Once sbt-scalafmt is at version > 2.x, use scalafmtAll addCommandAlias("format", ";scalafixAll OrganizeImports;scalafmt;test:scalafmt") @@ -78,7 +79,6 @@ addCommandAlias("format", ";scalafixAll OrganizeImports;scalafmt;test:scalafmt") ThisBuild / scalacOptions ++= Seq( "-deprecation", "-feature", - "-Ywarn-unused", // required by scalafix "-Xfatal-warnings", "-language:implicitConversions", "-Ycache-macro-class-loader:last-modified", diff --git a/performance/build.sbt b/performance/build.sbt new file mode 100644 index 000000000..c6ed3dcaa --- /dev/null +++ b/performance/build.sbt @@ -0,0 +1,6 @@ +name := "performance" + +dependsOn(Projects.semanticcpg) + +enablePlugins(JmhPlugin) + diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala new file mode 100644 index 000000000..8794f3f1f --- /dev/null +++ b/performance/src/main/scala/MyTests.scala @@ -0,0 +1,170 @@ +package io.shiftleft.semanticcpg.language.types.expressions + +import io.shiftleft.codepropertygraph.generated.NodeTypes +import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Expression, Method, MethodParameterIn, MethodReturn, StoredNode} +import io.shiftleft.semanticcpg.testing.MockCpg +import io.shiftleft.semanticcpg.language.MySteps._ +import io.shiftleft.semanticcpg.language.ImportsV3._ + +import scala.jdk.CollectionConverters._ +import org.openjdk.jmh.annotations._ +import MyTests._ +import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers + +import scala.collection.{IterableOnceOps, SeqOps, View, mutable} +import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import org.openjdk.jmh.infra.Blackhole +import overflowdb.traversal.Traversal +import some.SomeDomain._ + +object MyTests { + import io.shiftleft.semanticcpg.language._ + @State(Scope.Benchmark) + class MyState { + val (method, local) = { + val cpg = MockCpg() + .withMethod("methodForCfgTest") + .withLocalInMethod("methodForCfgTest", "x") + .withCallInMethod("methodForCfgTest", "call1") + .withCallInMethod("methodForCfgTest", "call2") + .withIdentifierArgument("call1", "x") + .cpg + + (cpg.method.name("methodForCfgTest").head, cpg.local.name("x").head) + } + val d1 = D1(D2()) + } +} + +case class Bar[T](x: T) + +class MyTestNew { + import io.shiftleft.semanticcpg.langv2._ + @Benchmark + def travNew(state: MyState) = { + val x = state.method.methodReturn + x + } + + @Benchmark + def travBase(state: MyState) = { + val x = state.method._astOut.asScala.collectFirst { case x: MethodReturn => x }.get + x + } + + @Benchmark + def refIdNew2(state: MyState) = { + val x = toLocalTraversalSingle(state.local).referencingIdentifiers + x + } + + @Benchmark + def refIdNew3(state: MyState) = { + val x = Seq.from(rftoSingleExt(state.local).referencingIdentifiers) + x + } + + @Benchmark + def refIdBase(state: MyState) = { + val x = Seq.from(state.local._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER)) + x + } + + @Benchmark + def allocTime1() = { + val x = mutable.ArrayBuffer.empty[Int] + x.append(1) + x + } + + @Benchmark + def allocTime2() = { + Some(1) + } + + @Benchmark + def allocTime3() = { + val x = ListBuffer.empty[Int] + x.append(1) + x + } + + @Benchmark + def astTestNewV2(state: MyState) = { + val x: Option[Expression] = toAstNodeTraversalSingle(state.method).isExpression + x + } + + @Benchmark + def astTestNewV3(state: MyState) = { + toSingleExt(state.method).isExpression + } + + @Benchmark + def astAstNew(state: MyState) = { + state.method.ast + } + + @Benchmark + def syntheticNew(state: MyState) = { + state.d1.toD2 + } + + @Benchmark + def syntheticBase(state: MyState) = { + state.d1.x + } + + @Benchmark + def syntheticIterableNew(state: MyState) = { + //toSynth(state.d1:: Nil).toD2Multi + //val y: TravOps[Iterable, IterableTypes[Iterable, Iterable[Any]]] = toIt2Ops + val x: List[D2] = List(state.d1).toD2 + x + } + + @Benchmark + def syntheticIterableBase(state: MyState) = { + Iterable.single(state.d1).map(_.x) + } + + def compileTest(state: MyState) = { + val a: D2 = state.d1.toD2 + val b: Option[D2] = Option(state.d1).toD2 + val c = toSynthIter(Array(state.d1).view.slice(1,2)).toD2 + val d: View[D2] = c + val z: scala.collection.Seq[Int] = ArrayBuffer.empty[Int] + + Iterable.single(state.d1).toD2Multi + Iterator.single(state.d1).toD2Multi + + val a1 = Iterable.single(state.d1).doGlobal + val a2 = Iterator.single(state.d1).doGlobal + } +} + +class MyTestsOld { + import io.shiftleft.semanticcpg.language._ + + @Benchmark + def travOld(state: MyState) = { + val x = toMethodMethods(state.method).methodReturn.l + x + } + + @Benchmark + def refIdOld(state: MyState) = { + val x = Traversal(state.local).referencingIdentifiers2.toSeq + x + } + + @Benchmark + def astTestOld(state: MyState) = { + Traversal(state.method).isExpression.headOption + } + + @Benchmark + def astAstOld(state: MyState) = { + Traversal(state.method).ast.l + } +} diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala new file mode 100644 index 000000000..d94268306 --- /dev/null +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -0,0 +1,67 @@ +package some + +import io.shiftleft.semanticcpg.langv2._ +import some.SomeDomain.D1 + +import scala.collection.IterableOnceOps + + +object SomeDomain { + implicit def toSynthSingle[I <: D1](p: I): SynthExt[I, Single, Nothing] = { + new SynthExt(p: Single[I]) + } + + implicit def toSynthOption[I <: D1](trav: Option[I]): SynthExt[I, Option, Nothing] = { + new SynthExt(trav) + } + + implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]) + : SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + } + + class SynthExt[I <: D1, IT[_], Marker](val in: IT[I]) extends AnyVal { + def toD2(implicit applier: ToOne[IT, Marker]) = { + applier.apply(in)(_.x) + } + def toD2Multi(implicit applier: ToMany[IT, Marker]) = { + applier.apply(in)(Iterator.single) + } + def doGlobal(implicit applier: ToGlobal[IT, Marker]) = { + applier.apply(in)(x => x) + } + } + + case class D1(x: D2) + + case class D2() +} +/* +object SomeDomain2 { + trait SyncExtImplicits { + implicit def toSynthExtSingle(p: D1) = { + new SynthExt[Single, Single, Option, Option, Seq](p: Single[D1]) + } + + implicit def toSynthExtOption[T[_]](trav: Option[D1]) = { + new SynthExt[Option, Option, Option, Option, Seq](trav) + } + + implicit def toSynthExtIter[CC[_], C](trav: IterableOnceOps[D1, CC, C]) + : SynthExt[({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC] = { + new SynthExt[({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) + } + } + + class SynthExt[Collection[_], CCOneToOne[_], CCOneToOption[_], CCOneToBoolean[_], CCOneToMany[_]] + (val trav: Collection[D1]) extends AnyVal { + def toD2(implicit ops: TravOpsToOne[Collection]) = { + trav.map(_.x) + } + + def toD2Multi(implicit ops: TravOpsToOne[Collection]) = { + trav.flatMap(Iterator.single) + } + } +} + */ \ No newline at end of file diff --git a/project/Projects.scala b/project/Projects.scala index 228f7db69..77ddc7c26 100644 --- a/project/Projects.scala +++ b/project/Projects.scala @@ -8,4 +8,5 @@ object Projects { lazy val semanticcpg = project.in(file("semanticcpg")) lazy val macros = project.in(file("macros")) lazy val schema2json = project.in(file("schema2json")) + lazy val performance = project.in(file("performance")) } diff --git a/project/plugins.sbt b/project/plugins.sbt index 0b7c329cc..7fa616d88 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -8,3 +8,4 @@ addSbtPlugin("com.dwijnand" % "sbt-dynver" % "3.1.0") addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.5") addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.27") addSbtPlugin("io.shiftleft" % "sbt-overflowdb" % "2.9") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/ImportsV3.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/ImportsV3.scala new file mode 100644 index 000000000..02d62e94e --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/ImportsV3.scala @@ -0,0 +1,8 @@ +package io.shiftleft.semanticcpg.language + +import io.shiftleft.semanticcpg.language.types.expressions.generalizations.AstNodeIsExpression +import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers + +object ImportsV3 extends AstNodeIsExpression.Imports with LocalReferencingIdentifiers.Imports { + +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/MySteps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/MySteps.scala new file mode 100644 index 000000000..eef87f4dc --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/MySteps.scala @@ -0,0 +1,79 @@ +package io.shiftleft.semanticcpg.language + +import io.shiftleft.semanticcpg.language.MySteps._ +import io.shiftleft.codepropertygraph.generated.nodes +import io.shiftleft.semanticcpg.language.types.structure.LocalTraversalNew3 +import overflowdb.traversal.Traversal + +import scala.jdk.CollectionConverters._ + +// First attempt of new pipeline. Not further used since it is not so fast. + +trait EPipe[I] { + type ColT[_] + protected val p: ColT[I] + protected val ops: PipeOps[ColT] + + def head: I = { + ops.head(p) + } + + def map[O](f: I => O): EPipe[O] = { + new Pipe(ops.map(p)(f), ops.T1to1FollowUpOps) + } + + def filter2(f: I => Boolean): EPipe[I] = { + new Pipe(ops.filter(p)(f), ops.T1toOptionFollowUpOps) + } + + def flatMap[O](f: I => Iterable[O]): EPipe[O] = { + new Pipe(ops.flatMap(p)(f), ops.T1toNFollowUpOps) + } + + def flatMapIter[O](f: I => Iterator[O]): EPipe[O] = { + new Pipe(ops.flatMapIter(p)(f), ops.T1toNFollowUpOps) + } + + def cast[A <: I]: EPipe[A] = { + this.asInstanceOf[EPipe[A]] + } +} + +class Pipe[I, T[_]](val p: T[I], val ops: PipeOps[T]) extends EPipe[I] { + override type ColT[A] = T[A] +} + +object MySteps { + type Pipe1[T] = T + + implicit val ops1 = Pipe1Ops + + implicit val opsN = PipeNIterableOps + + implicit def toExtClass[I <: nodes.Method](pipe: Iterable[I]): ExtensionClass[I] = { + new ExtensionClass(new Pipe(pipe, PipeNIterableOps)) + } + + implicit def toExtClass[I <: nodes.Method](pipe: I): ExtensionClass[I] = { + new ExtensionClass(new Pipe(pipe: Pipe1[I], Pipe1Ops)) + } + + implicit def toExtClass[I <: nodes.Method](pipe: EPipe[I]): ExtensionClass[I] = { + new ExtensionClass(pipe) + } + + implicit def toLocalNew[I <: nodes.Local](pipe: I): LocalTraversalNew3 = { + new LocalTraversalNew3(new Pipe(pipe: Pipe1[I], Pipe1Ops)) + } + +} + +class ExtensionClass[I <: nodes.Method](val pipe: EPipe[I]) extends AnyVal { + def methodReturn2() = { + pipe.map(_._astOut.asScala.collectFirst { case x: nodes.MethodReturn => x }.get) + } + + def methodParameters = { + pipe.flatMapIter(_._astOut.asScala.collect { case x: nodes.MethodParameterIn => x }) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/PipeOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/PipeOps.scala new file mode 100644 index 000000000..7dca00684 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/PipeOps.scala @@ -0,0 +1,123 @@ +package io.shiftleft.semanticcpg.language + +import io.shiftleft.semanticcpg.language.MySteps.Pipe1 + +// PipeOps for the new pipeline attempt in MySteps.scala. + +trait PipeOps[PipeT[_]] { + type T1to1[T] + type T1toOption[T] + type T1toN[T] + val T1to1FollowUpOps: PipeOps[T1to1] + val T1toOptionFollowUpOps: PipeOps[T1toOption] + val T1toNFollowUpOps: PipeOps[T1toN] + def head[I](pipe: PipeT[I]): I + def map[I, O](pipe: PipeT[I])(f: I => O): T1to1[O] + def filter[I](pipe: PipeT[I])(f: I => Boolean): T1toOption[I] + def flatMap[I, O](pipe: PipeT[I])(f: I => Iterable[O]): T1toN[O] + def flatMapIter[I, O](pipe: PipeT[I])(f: I => Iterator[O]): T1toN[O] +} + +object Pipe1Ops extends PipeOps[Pipe1] { + override type T1to1[T] = Pipe1[T] + override type T1toOption[T] = Option[T] + override type T1toN[T] = Iterable[T] + + override val T1to1FollowUpOps: PipeOps[T1to1] = Pipe1Ops + override val T1toOptionFollowUpOps: PipeOps[T1toOption] = PipeOptionOps + override val T1toNFollowUpOps: PipeOps[Iterable] = PipeNIterableOps + + override def head[I](pipe: Pipe1[I]): I = { + pipe + } + + override def map[I, O](pipe: Pipe1[I])(f: I => O): T1to1[O] = { + f(pipe) + } + + override def filter[I](pipe: Pipe1[I])(f: I => Boolean): T1toOption[I] = { + if (f(pipe)) { + Some(pipe) + } else { + None + } + } + + override def flatMap[I, O](pipe: Pipe1[I])(f: I => Iterable[O]): T1toN[O] = { + Iterable.from(f(pipe)) + } + + override def flatMapIter[I, O](pipe: Pipe1[I])(f: I => Iterator[O]): T1toN[O] = { + Iterable.from(f(pipe)) + } +} + +object PipeNIterableOps extends PipeOps[Iterable] { + override type T1to1[T] = Iterable[T] + override type T1toOption[T] = Iterable[T] + override type T1toN[T] = Iterable[T] + + override val T1to1FollowUpOps: PipeOps[T1to1] = PipeNIterableOps + override val T1toOptionFollowUpOps: PipeOps[T1toOption] = PipeNIterableOps + override val T1toNFollowUpOps: PipeOps[Iterable] = PipeNIterableOps + + override def head[I](pipe: Iterable[I]): I = { + pipe.head + } + + override def map[I, O](pipe: Iterable[I])(f: I => O): T1to1[O] = { + pipe.map(f) + } + + override def filter[I](pipe: Iterable[I])(f: I => Boolean): T1toOption[I] = { + pipe.filter(f) + } + + override def flatMap[I, O](pipe: Iterable[I])(f: I => Iterable[O]): T1toN[O] = { + pipe.flatMap(f) + } + + override def flatMapIter[I, O](pipe: Iterable[I])(f: I => Iterator[O]): T1toN[O] = { + pipe.flatMap(f) + } +} + +object PipeOptionOps extends PipeOps[Option] { + override type T1to1[T] = Option[T] + override type T1toOption[T] = Option[T] + override type T1toN[T] = Iterable[T] + + override val T1to1FollowUpOps: PipeOps[T1to1] = PipeOptionOps + override val T1toOptionFollowUpOps: PipeOps[T1toOption] = PipeOptionOps + override val T1toNFollowUpOps: PipeOps[Iterable] = PipeNIterableOps + + override def head[I](pipe: Option[I]): I = { + pipe.get + } + + override def map[I, O](pipe: Option[I])(f: I => O): T1to1[O] = { + pipe.map(f) + } + + override def filter[I](pipe: Option[I])(f: I => Boolean): T1toOption[I] = { + pipe.filter(f) + } + + override def flatMap[I, O](pipe: Option[I])(f: I => Iterable[O]): T1toN[O] = { + pipe match { + case Some(p) => + f(p) + case None => + Iterable.empty + } + } + + override def flatMapIter[I, O](pipe: Option[I])(f: I => Iterator[O]): T1toN[O] = { + pipe match { + case Some(p) => + Iterable.from(f(p)) + case None => + Iterable.empty + } + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index a6c84d8eb..26107f2d7 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -1,11 +1,53 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, nodes} import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.langv3.{Helper, Kernel1ToN, Kernel1ToO} import overflowdb.traversal.help.Doc import overflowdb.traversal.{Traversal, help} +import scala.collection.{IterableOnceOps, IterableOps} +import scala.jdk.CollectionConverters._ + +class AstNodeIsExpressionKernel[I <: AstNode] extends Kernel1ToO[I, I with Expression] { + override def apply(i: I): Option[I with Expression] = { + if (i.isInstanceOf[Expression]) { + Some(i.asInstanceOf[I with Expression]) + } else { + None + } + } +} + +object AstNodeIsExpression { + type NodeType = AstNode + type KernelType[T <: NodeType] = AstNodeIsExpressionKernel[T] + + private val _impl = new KernelType[NodeType]() + private def impl[I <: NodeType] = _impl.asInstanceOf[KernelType[I]] + + trait Imports { + implicit def toSingleExt[I <: NodeType](i: I) = new SingleExt(i) + implicit def toIterableExt[I <: NodeType, CC[_], C](i: IterableOps[I, CC, C]) = new IterableExt(i) + implicit def toIteratorExt[I <: NodeType](i: Iterator[I]) = new IteratorExt(i) + implicit def toOptionExt[I <: NodeType](i: Option[I]) = new OptionExt(i) + } + + class SingleExt[I <: NodeType](val i: I) extends AnyVal { + def isExpression = Helper(i, impl[I]) + } + class IterableExt[I <: NodeType, CC[_], C](val i: IterableOps[I, CC, C]) extends AnyVal { + def isExpression = Helper(i, impl[I]) + } + class IteratorExt[I <: NodeType](val i: Iterator[I]) extends AnyVal { + def isExpression = Helper(i, impl[I]) + } + class OptionExt[I <: NodeType](val i: Option[I]) extends AnyVal { + def isExpression = Helper(i, impl[I]) + } +} + @help.Traversal(elementType = classOf[AstNode]) class AstNodeTraversal[A <: AstNode](val traversal: Traversal[A]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 7beb8fa90..c9333c66b 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -1,9 +1,50 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.semanticcpg.language.MySteps._ +import io.shiftleft.semanticcpg.language.EPipe +import io.shiftleft.semanticcpg.langv3.{Helper, Kernel1ToN, Kernel1ToO} import overflowdb.traversal.Traversal +import scala.collection.IterableOps +import scala.jdk.CollectionConverters._ + +class LocalReferencingIdentifiersKernel[I <: nodes.Local] extends Kernel1ToN[I, nodes.Identifier] { + override def apply(i: I): Iterator[nodes.Identifier] = { + i._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER).asInstanceOf[Iterator[nodes.Identifier]] + } +} + +object LocalReferencingIdentifiers { + type NodeType = nodes.Local + type KernelType[T <: NodeType] = LocalReferencingIdentifiersKernel[T] + + private val _impl = new KernelType[NodeType]() + private def impl[I <: NodeType] = _impl.asInstanceOf[KernelType[I]] + + trait Imports { + implicit def rftoSingleExt[I <: NodeType](i: I) = new SingleExt(i) + implicit def rftoIterableExt[I <: NodeType, CC[_], C](i: IterableOps[I, CC, C]) = new IterableExt(i) + implicit def rftoIteratorExt[I <: NodeType](i: Iterator[I]) = new IteratorExt(i) + implicit def rftoOptionExt[I <: NodeType](i: Option[I]) = new OptionExt(i) + } + + class SingleExt[I <: NodeType](val i: I) extends AnyVal { + def referencingIdentifiers = Helper(i, impl[I]) + } + class IterableExt[I <: NodeType, CC[_], C](val i: IterableOps[I, CC, C]) extends AnyVal { + def referencingIdentifiers = Helper(i, impl[I]) + } + class IteratorExt[I <: NodeType](val i: Iterator[I]) extends AnyVal { + def referencingIdentifiers = Helper(i, impl[I]) + } + class OptionExt[I <: NodeType](val i: Option[I]) extends AnyVal { + def referencingIdentifiers = Helper(i, impl[I]) + } +} + /** * A local variable * */ @@ -13,7 +54,7 @@ class LocalTraversal(val traversal: Traversal[Local]) extends AnyVal { * The method hosting this local variable * */ def method: Traversal[Method] = { - // TODO The following line of code is here for backwards compatibility. + // FTDO The following line of code is here for backwards compatibility. // Use the lower commented out line once not required anymore. traversal.repeat(_.in(EdgeTypes.AST))(_.until(_.hasLabel(NodeTypes.METHOD))).cast[Method] //definingBlock.method @@ -31,6 +72,9 @@ class LocalTraversal(val traversal: Traversal[Local]) extends AnyVal { def referencingIdentifiers: Traversal[Identifier] = traversal.in(EdgeTypes.REF).hasLabel(NodeTypes.IDENTIFIER).cast[Identifier] + def referencingIdentifiers2: Traversal[Identifier] = + traversal.flatMap(_._refIn.asScala).filter(_.label == NodeTypes.IDENTIFIER).cast[Identifier] + /** * The type of the local. * @@ -39,3 +83,16 @@ class LocalTraversal(val traversal: Traversal[Local]) extends AnyVal { def typ: Traversal[Type] = traversal.out(EdgeTypes.EVAL_TYPE).cast[Type] } + +class LocalTraversalNew3(val nodeOrPipe: EPipe[Local]) extends AnyVal { + def definingBlock = { + nodeOrPipe.map(_._astIn.asScala.next()).cast[Block] + } + + def referencingIdentifiers = { + nodeOrPipe + .flatMapIter(_._refIn.asScala) + .filter2(_.label == NodeTypes.IDENTIFIER) + .cast[Identifier] + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala new file mode 100644 index 000000000..06e6edd17 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -0,0 +1,50 @@ +package io.shiftleft.semanticcpg.langv2 + +import overflowdb.traversal.RepeatStepIterator + +import scala.collection.IterableOnceOps + +trait AnyTraversalImplicits { + implicit def toAnyTraversalIter[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + new AnyTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + } + implicit def toAnyTraversalInternal[I, IT[_], Marker](trav: IT[I]) = { + new AnyTraversal[I, IT, Marker](trav) + } +} + +class AnyTraversal[I, IT[_], Marker](val trav: IT[I]) extends AnyVal { + + def cast[A](implicit applier: ToOne[IT, Marker]): applier.OUT[A] = { + trav.asInstanceOf[applier.OUT[A]] + } + + // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. + def collectAll[T](implicit applier: ToBoolean[IT, Marker], isInstanceOfOps: IsInstanceOfOps[T]): applier.OUT[T] = { + applier.apply(trav)(isInstanceOfOps).asInstanceOf[applier.OUT[T]] + } + + def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( + implicit applier: ToMany[IT, Marker]): applier.OUT[I] = { + val behaviour = g(new RepeatBehaviourBuilder()).build + applier.apply(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) + } + + def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( + implicit applier: ToMany[IT, Marker]): applier.OUT[I] = { + val behaviour = g(new RepeatBehaviourBuilder()).build + applier.apply(trav)(i => new RepeatStepIterator(i, f, behaviour)) + } + + //@deprecated("Use rFlatMap instead") + def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( + implicit applier: ToMany[IT, Marker]): applier.OUT[I] = { + rFlatMap(f, g) + } + + @deprecated("Calls to this function can be omitted") + def l: IT[I] = { + trav + } + +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/Example.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/Example.scala new file mode 100644 index 000000000..bdb1f8046 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/Example.scala @@ -0,0 +1,51 @@ +package io.shiftleft.semanticcpg.langv2 + +object Example { + trait TypeMultiplexer { + type IN[_] + type OUT1[_] + type OUT2[_] + } + + abstract final class TypeGroup1 extends TypeMultiplexer { + override type IN[T] = Option[T] + override type OUT1[T] = Option[T] + override type OUT2[T] = Iterable[T] + } + + trait Ops[_TM <: TypeMultiplexer] { + type TM = _TM + def map[I, O](a: TM#IN[I])(f: I => O): TM#OUT1[O] + def flatMap[I, O](a: TM#IN[I])(f: I => Iterator[O]): TM#OUT2[O] + } + + implicit object TypeGroup1Ops extends Ops[TypeGroup1] { + override def map[I, O](a: this.TM#IN[I])(f: I => O): this.TM#OUT1[O] = { + a.map(f) + } + + override def flatMap[I, O](a: this.TM#IN[I])(f: I => Iterator[O]): this.TM#OUT2[O] = { + a.map(f.andThen(Iterable.from)).getOrElse(Iterable.empty) + } + } + + case class Dom1(x: Iterable[Dom2]) + case class Dom2() + + implicit def toSomeExtension(i: Option[Dom1]) = { + new SomeExtension[TypeGroup1](i) + } + + class SomeExtension[TM <: TypeMultiplexer](val a: TM#IN[Dom1]) extends AnyVal { + def toFirstDom2(implicit ops: Ops[TM]) = { + ops.map(a)(_.x.head) + } + def toAllDom2(implicit ops: Ops[TM]) = { + ops.flatMap(a)(_.x.iterator) + } + } + + val d1 = Dom1(Dom2():: Dom2():: Nil) + Option(d1).toFirstDom2 + +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/ExtensionClassImplicits.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/ExtensionClassImplicits.scala new file mode 100644 index 000000000..9db990908 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/ExtensionClassImplicits.scala @@ -0,0 +1,9 @@ +package io.shiftleft.semanticcpg.langv2 + +import io.shiftleft.semanticcpg.langv2.types.expressions.generalizations.AstNodeTraversalImplicits +import io.shiftleft.semanticcpg.langv2.types.structure.{LocalTraversalImplicits, MethodTraversalImplicits} + +trait ExtensionClassImplicits + extends MethodTraversalImplicits + with AstNodeTraversalImplicits + with LocalTraversalImplicits diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala new file mode 100644 index 000000000..f7db7977c --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala @@ -0,0 +1,12 @@ +package io.shiftleft.semanticcpg.langv2 + +import io.shiftleft.codepropertygraph.generated.nodes + +trait IsInstanceOfOps[T] extends Function1[Any, Boolean] + +trait InstanceOfOpsImplicits { + implicit val methodReturnCollectOps: IsInstanceOfOps[nodes.MethodReturn] = _.isInstanceOf[nodes.MethodReturn] + implicit val methodParameterCollectOps: IsInstanceOfOps[nodes.MethodParameterIn] = _.isInstanceOf[nodes.MethodParameterIn] + implicit val astNodeCollectOps: IsInstanceOfOps[nodes.AstNode] = _.isInstanceOf[nodes.AstNode] + implicit val expressionCollectOps: IsInstanceOfOps[nodes.Expression] = _.isInstanceOf[nodes.Expression] +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/RepeatBehaviourBuilder.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/RepeatBehaviourBuilder.scala new file mode 100644 index 000000000..2bfd65b8c --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/RepeatBehaviourBuilder.scala @@ -0,0 +1,77 @@ +package io.shiftleft.semanticcpg.langv2 + +import overflowdb.traversal.RepeatBehaviour +import overflowdb.traversal.RepeatBehaviour.SearchAlgorithm + +class RepeatBehaviourBuilder[T] { + private[this] var _shouldEmit: (T, Int) => Boolean = (_, _) => false + private[this] var _untilCondition: Option[T => Iterator[_]] = None + private[this] var _whileCondition: Option[T => Iterator[_]] = None + private[this] var _times: Option[Int] = None + private[this] var _dedupEnabled: Boolean = false + private[this] var _searchAlgorithm: SearchAlgorithm.Value = SearchAlgorithm.DepthFirst + + /* configure search algorithm to go "breadth first", rather than the default "depth first" */ + def breadthFirstSearch: RepeatBehaviourBuilder[T] = { + _searchAlgorithm = SearchAlgorithm.BreadthFirst + this + } + + def bfs: RepeatBehaviourBuilder[T] = breadthFirstSearch + + /* configure `repeat` step to emit everything along the way */ + def emit: RepeatBehaviourBuilder[T] = { + _shouldEmit = (_, _) => true + this + } + + /* configure `repeat` step to emit everything along the way, apart from the _first_ element */ + def emitAllButFirst: RepeatBehaviourBuilder[T] = { + _shouldEmit = (_, depth) => depth > 0 + this + } + + /* configure `repeat` step to emit whatever meets the given condition */ + def emit(condition: T => Iterator[_]): RepeatBehaviourBuilder[T] = { + _shouldEmit = (element, _) => condition(element).hasNext + this + } + + /* Configure `repeat` step to stop traversing when given condition-traversal has at least one result. + * The condition-traversal is only evaluated _after_ the first iteration, for classic repeat/until behaviour */ + def until(condition: T => Iterator[_]): RepeatBehaviourBuilder[T] = { + _untilCondition = Some(condition) + this + } + + /* Configure `repeat` step to stop traversing when given condition-traversal has at least one result. + * The condition-traversal is already evaluated at the first iteration, for classic while/repeat behaviour. + * + * n.b. the only reason not to call this `while` is to avoid using scala keywords, which would need to be quoted. */ + def whilst(condition: T => Iterator[_]): RepeatBehaviourBuilder[T] = { + _whileCondition = Some(condition) + this + } + + /* configure `repeat` step to perform the given amount of iterations */ + def times(value: Int): RepeatBehaviourBuilder[T] = { + _times = Some(value) + this + } + + def dedup: RepeatBehaviourBuilder[T] = { + _dedupEnabled = true + this + } + + def build: RepeatBehaviour[T] = { + new RepeatBehaviour[T] { + override val searchAlgorithm: SearchAlgorithm.Value = _searchAlgorithm + override val untilCondition = _untilCondition + override val whileCondition = _whileCondition + final override val times: Option[Int] = _times + final override val dedupEnabled = _dedupEnabled + override def shouldEmit(element: T, currentDepth: Int): Boolean = _shouldEmit(element, currentDepth) + } + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala new file mode 100644 index 000000000..4e2a918ed --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -0,0 +1,205 @@ +package io.shiftleft.semanticcpg.langv2 + +import scala.collection.IterableOnceOps +import scala.collection.mutable.ArrayBuffer +import scala.collection.Seq + +class IterTypes[_CC[T], _C] { + type CC[T] = _CC[T] + type C = _C +} + +trait ToOne[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] + + def apply[I, O](in: IN[I])(f: I => O): OUT[O] +} + +object SingleToOne extends ToOne[Single, Nothing] { + override type OUT[T] = T + + override def apply[I, O](in: IN[I])(f: I => O): OUT[O] = f(in) +} + +object OptionToOne extends ToOne[Option, Nothing] { + override type OUT[T] = Option[T] + + override def apply[I, O](in: IN[I])(f: I => O): OUT[O] = in.map(f) +} + +class IterToOne[CC[_], C] extends ToOne[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = CC[T] + + override def apply[I, O](in: IN[I])(f: I => O): OUT[O] = in.map(f) +} + +trait ToBoolean[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] + + def apply[I, O](in: IN[I])(f: I => Boolean): OUT[I] +} + +object SingleToBoolean extends ToBoolean[Single, Nothing] { + override type OUT[T] = Option[T] + + override def apply[I, O](in: IN[I])(f: I => Boolean): OUT[I] = + if (f(in)) { + Some(in) + } else { + None + } +} + +object OptionToBoolean extends ToBoolean[Option, Nothing] { + override type OUT[T] = Option[T] + + override def apply[I, O](in: IN[I])(f: I => Boolean): OUT[I] = in.filter(f) +} + +class IterToBoolean[CC[_], C] extends ToBoolean[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = C + + override def apply[I, O](in: IN[I])(f: I => Boolean): OUT[I] = in.filter(f) +} + +trait ToOption[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] + + def apply[I, O](in: IN[I])(f: I => Option[O]): OUT[O] +} + +object SingleToOption extends ToOption[Single, Nothing] { + override type OUT[T] = Option[T] + + override def apply[I, O](in: IN[I])(f: I => Option[O]): OUT[O] = f(in) +} + +object OptionToOption extends ToOption[Option, Nothing] { + override type OUT[T] = Option[T] + + override def apply[I, O](in: IN[I])(f: I => Option[O]): OUT[O] = in.flatMap(f) +} + +class IterToOption[CC[_], C] extends ToOption[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = CC[T] + + override def apply[I, O](in: IN[I])(f: I => Option[O]): OUT[O] = in.flatMap(f) +} + +trait ToMany[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] + + def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] +} + +object SingleToMany extends ToMany[Single, Nothing] { + override type OUT[T] = Seq[T] + + override def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] = Seq.from(f(in)) +} + +object OptionToMany extends ToMany[Option, Nothing] { + override type OUT[T] = Seq[T] + + override def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] = + in match { + case Some(i) => + Seq.from(f(i)) + case None => + Seq.empty + } +} + +class IterToMany[CC[_], C] + extends ToMany[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = CC[T] + + override def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] = { + in.flatMap(f) + } +} + +trait ToGlobal[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] + + def apply[I, O](in: IN[I])(f: collection.Seq[I] => collection.Seq[O]): OUT[O] +} + +object SingleToGlobal extends ToGlobal[Single, Nothing] { + override type OUT[T] = collection.Seq[T] + + override def apply[I, O](in: IN[I])(f: collection.Seq[I] => collection.Seq[O]): OUT[O] = { + f(in::Nil) + } +} + +object OptionToGlobal extends ToGlobal[Option, Nothing] { + override type OUT[T] = collection.Seq[T] + + override def apply[I, O](in: IN[I])(f: collection.Seq[I] => collection.Seq[O]): OUT[O] = { + in match { + case Some(i) => + f(i::Nil) + case None => + f(Nil) + } + } +} + +class IterToGlobal[CC[_], C] extends ToGlobal[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = CC[T] + + override def apply[I, O](in: IN[I])(f: collection.Seq[I] => collection.Seq[O]): OUT[O] = { + in match { + case iterable: Iterable[I] => + iterable.iterableFactory.from(f(iterable.toSeq)).asInstanceOf[CC[O]] + case iterator: Iterator[I] => + new Iterator[O] { + private var resultIt: Iterator[O] = _ + override def hasNext: Boolean = { + if (resultIt == null) { + resultIt = f(in.to(ArrayBuffer)).iterator + } + resultIt.hasNext + } + override def next(): O = { + resultIt.next() + } + }.asInstanceOf[CC[O]] + } + } +} + +trait ToGlobalEnd[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + + def apply[I, O](in: IN[I])(f: Seq[I] => O): O +} + +object SingleToGlobalEnd extends ToGlobalEnd[Single, Nothing] { + override def apply[I, O](in: IN[I])(f: Seq[I] => O): O = { + f(in::Nil) + } +} + +object OptionToGlobalEnd extends ToGlobalEnd[Option, Nothing] { + override def apply[I, O](in: IN[I])(f: Seq[I] => O): O = { + in match { + case Some(i) => + f(i::Nil) + case None => + f(Nil) + } + } +} + +class IterToGlobalEnd[CC[_], C] extends ToGlobalEnd[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override def apply[I, O](in: IN[I])(f: Seq[I] => O): O = { + f(in.toSeq) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala new file mode 100644 index 000000000..071580c9e --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala @@ -0,0 +1,2 @@ +package io.shiftleft.semanticcpg.langv2 + diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala new file mode 100644 index 000000000..6c95d184d --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -0,0 +1,33 @@ +package io.shiftleft.semanticcpg + +import scala.collection.IterableOnceOps + +package object langv2 extends ExtensionClassImplicits with AnyTraversalImplicits with InstanceOfOpsImplicits { + type Single[T] = T + + implicit val singleToOne: SingleToOne.type = SingleToOne + implicit val optionToOne: OptionToOne.type = OptionToOne + private val _iterToOne = new IterToOne() + implicit def iterToOne[CC[_], C]: IterToOne[CC, C] = _iterToOne.asInstanceOf[IterToOne[CC, C]] + + implicit val singleToBoolean: SingleToBoolean.type = SingleToBoolean + implicit val optionToBoolean: OptionToBoolean.type = OptionToBoolean + private val _iterToBoolean = new IterToBoolean() + implicit def iterToBoolean[CC[_], C]: IterToBoolean[CC, C] = _iterToBoolean.asInstanceOf[IterToBoolean[CC, C]] + + implicit val singleToOption: SingleToOption.type = SingleToOption + implicit val optionToOption: OptionToOption.type = OptionToOption + private val _iterToOption = new IterToOption() + implicit def iterToOption[CC[_], C]: IterToOption[CC, C] = _iterToOption.asInstanceOf[IterToOption[CC, C]] + + implicit val singleToMany: SingleToMany.type = SingleToMany + implicit val optionToMany: OptionToMany.type = OptionToMany + private val _iterToMany = new IterToMany() + implicit def iterToMany[CC[_], C]: IterToMany[CC, C] = _iterToMany.asInstanceOf[IterToMany[CC, C]] + + implicit val singleToGlobal: SingleToGlobal.type = SingleToGlobal + implicit val optionToGlobal: OptionToGlobal.type = OptionToGlobal + private val _iterToGlobal = new IterToGlobal() + implicit def iterToGlobal[CC[_], C]: IterToGlobal[CC, C] = _iterToGlobal.asInstanceOf[IterToGlobal[CC, C]] + +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala new file mode 100644 index 000000000..be809e768 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -0,0 +1,43 @@ +package io.shiftleft.semanticcpg.langv2.types.expressions.generalizations + +import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Expression} +import io.shiftleft.semanticcpg.langv2._ +import overflowdb.traversal.help.Doc + +import scala.collection.IterableOnceOps +import scala.jdk.CollectionConverters._ + + +trait AstNodeTraversalImplicits { + implicit def toAstNodeTraversalSingle[I <: AstNode](in: I): AstNodeTraversal[I, Single, Nothing] = { + new AstNodeTraversal(in: Single[I]) + } + implicit def toAstNodeTraversalOption[I <: AstNode](in: Option[I]): AstNodeTraversal[I, Option, Nothing] = { + new AstNodeTraversal(in) + } + implicit def toAstNodeTraversalIter[I <: AstNode, CC[_], C](in: IterableOnceOps[I, CC, C]) + : AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](in) + } + +} + +class AstNodeTraversal[I <: AstNode, IT[_], Marker](val in: IT[I]) + extends AnyVal { + + /** + * Nodes of the AST rooted in this node, including the node itself. + * */ + @Doc("All nodes of the abstract syntax tree") + def ast(implicit applier: ToMany[IT, Marker]) = { + in.repeat(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) + } + + /** + * Traverse only to AST nodes that are expressions + * */ + def isExpression(implicit applier: ToBoolean[IT, Marker]) = + in.collectAll[Expression] + +} + diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala new file mode 100644 index 000000000..527d1f331 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -0,0 +1,28 @@ +package io.shiftleft.semanticcpg.langv2.types.structure + +import io.shiftleft.codepropertygraph.generated.nodes.Identifier +import io.shiftleft.codepropertygraph.generated.{NodeTypes, nodes} +import io.shiftleft.semanticcpg.langv2._ + +import scala.collection.IterableOnceOps +import scala.jdk.CollectionConverters._ + +trait LocalTraversalImplicits { + implicit def toLocalTraversalSingle[I <: nodes.Local](in: I): LocalTraversal[I, Single, Nothing] = { + new LocalTraversal(in: Single[I]) + } + implicit def toLocalTraversalOption[I <: nodes.Local](in: Option[I]): LocalTraversal[I, Option, Nothing] = { + new LocalTraversal(in) + } + implicit def toLocalTraversalIter[I <: nodes.Local, CC[_], C](in: IterableOnceOps[I, CC, C]) + : LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](in) + } +} + +class LocalTraversal[I <: nodes.Local, IT[_], Marker](val in: IT[I]) extends AnyVal { + def referencingIdentifiers(implicit applier: ToMany[IT, Marker]): applier.OUT[Identifier] = { + applier.apply(in)(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) + .cast[Identifier]) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala new file mode 100644 index 000000000..946702cc5 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -0,0 +1,50 @@ +package io.shiftleft.semanticcpg.langv2.types.structure + +import io.shiftleft.codepropertygraph.generated.nodes +import io.shiftleft.semanticcpg.langv2._ +import overflowdb.traversal.help +import overflowdb.traversal.help.Doc + +import scala.collection.IterableOnceOps +import scala.jdk.CollectionConverters._ + +trait MethodTraversalImplicits { + implicit def toMethodTraversalSingle[I <: nodes.Method](in: I): MethodTraversal[I, Single, Nothing] = { + new MethodTraversal(in: Single[I]) + } + implicit def toMethodTraversalOption[I <: nodes.Method](in: Option[I]): MethodTraversal[I, Option, Nothing] = { + new MethodTraversal(in) + } + implicit def toMethodTraversalIter[I <: nodes.Method, CC[_], C](in: IterableOnceOps[I, CC, C]) + : MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](in) + } +} + +/** + * A method, function, or procedure + * */ +@help.Traversal(elementType = classOf[nodes.Method]) +class MethodTraversal[I <: nodes.Method, IT[_], Marker](val in: IT[I]) extends AnyVal { + + /** + * Traverse to parameters of the method + * */ + @Doc("All parameters") + def parameter(implicit applier: ToMany[IT, Marker]) = { + applier.apply(in)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) + } + + /** + * Traverse to formal return parameter + * */ + @Doc("All formal return parameters") + def methodReturn(implicit applier: ToOne[IT, Marker]) = { + applier.apply(in)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) + } + + //def toParameterCount(implicit applier: ToGlobal[IT, Marker]) = { + // applier.apply(in)(_.map(_.parameter).size) + //} + +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv3/Kernel.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv3/Kernel.scala new file mode 100644 index 000000000..d5827f5e9 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv3/Kernel.scala @@ -0,0 +1,57 @@ +package io.shiftleft.semanticcpg.langv3 + +import scala.collection.{IterableOps} + +trait Kernel1To1[-I, +O] extends Function1[I, O] +trait Kernel1ToN[-I, +O] extends Function1[I, Iterator[O]] +trait Kernel1ToO[-I, +O] extends Function1[I, Option[O]] + +object Helper { + def apply[I, O](x: I, kernel: Kernel1To1[I, O]): O = { + kernel.apply(x) + } + + def apply[I, O](x: I, kernel: Kernel1ToN[I, O]): Iterator[O] = { + kernel.apply(x) + } + + def apply[I, O](x: I, kernel: Kernel1ToO[I, O]): Option[O] = { + kernel.apply(x) + } + + def apply[I, CC[_], C, O](x: IterableOps[I, CC, C], kernel: Kernel1To1[I, O]): CC[O] = { + x.map(kernel) + } + + def apply[I, CC[_], C, O](x: IterableOps[I, CC, C], kernel: Kernel1ToN[I, O]): CC[O] = { + x.flatMap(kernel) + } + + def apply[I, CC[_], C, O](x: IterableOps[I, CC, C], kernel: Kernel1ToO[I, O]): CC[O] = { + x.flatMap(kernel) + } + + def apply[I, O](x: Iterator[I], kernel: Kernel1To1[I, O]): Iterator[O] = { + x.map(kernel) + } + + def apply[I, O](x: Iterator[I], kernel: Kernel1ToN[I, O]): Iterator[O] = { + x.flatMap(kernel) + } + + def apply[I, O](x: Iterator[I], kernel: Kernel1ToO[I, O]): Iterator[O] = { + x.flatMap(kernel) + } + + def apply[I, O](x: Option[I], kernel: Kernel1To1[I, O]): Option[O] = { + x.map(kernel) + } + + def apply[I, O](x: Option[I], kernel: Kernel1ToN[I, O]): Iterable[O] = { + x.map(y => Iterable.from(kernel(y))).getOrElse(Iterable.empty) + } + + def apply[I, O](x: Option[I], kernel: Kernel1ToO[I, O]): Option[O] = { + x.flatMap(kernel) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala index d053c0aee..d9d9e106d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala @@ -178,12 +178,14 @@ package object testing { val callNode = cpg.call.name(callName).head val methodNode = callNode.method.head val identifierNode = NewIdentifier().name(name) + val localNode = cpg.local.name(name).head val typeDecl = NewTypeDecl().name("abc") graph.addNode(identifierNode) graph.addNode(typeDecl) graph.addEdge(callNode, identifierNode, EdgeTypes.AST) graph.addEdge(methodNode, identifierNode, EdgeTypes.CONTAINS) graph.addEdge(identifierNode, typeDecl, EdgeTypes.REF) + graph.addEdge(identifierNode, localNode, EdgeTypes.REF) } def withCustom(f: (DiffGraph.Builder, Cpg) => Unit): MockCpg = {