diff --git a/build.mill b/build.mill index ee3dafa131c..9b4d7acab28 100644 --- a/build.mill +++ b/build.mill @@ -16,6 +16,8 @@ import mill.resolve.SelectMode import mill.T import mill.define.Cross +import scala.util.matching.Regex + // plugins and dependencies import $meta._ import $file.ci.shared @@ -743,7 +745,7 @@ object dist0 extends MillPublishJavaModule { object dist extends MillPublishJavaModule { def jar = rawAssembly() - def moduleDeps = Seq(build.runner, idea) + def moduleDeps = Seq(build.runner, idea, build.main.init) def testTransitiveDeps = dist0.testTransitiveDeps() ++ Seq( (s"com.lihaoyi-${dist.artifactId()}", dist0.runClasspath().map(_.path).mkString("\n")) @@ -929,13 +931,20 @@ def bootstrapLauncher = T { PathRef(outputPath) } -def exampleZips: T[Seq[PathRef]] = T { +def examplePathsWithArtifactName:Task[Seq[(os.Path,String)]] = T.task{ for { exampleMod <- build.example.exampleModules - examplePath = exampleMod.millSourcePath + path = exampleMod.millSourcePath } yield { - val example = examplePath.subRelativeTo(T.workspace) - val exampleStr = millVersion() + "-" + example.segments.mkString("-") + val example = path.subRelativeTo(T.workspace) + val artifactName = millVersion() + "-" + example.segments.mkString("-") + (path, artifactName) + } +} + + +def exampleZips: T[Seq[PathRef]] = T{ + examplePathsWithArtifactName().map{ case (examplePath, exampleStr) => os.copy(examplePath, T.dest / exampleStr, createFolders = true) os.write(T.dest / exampleStr / ".mill-version", millLastTag()) os.copy(bootstrapLauncher().path, T.dest / exampleStr / "mill") diff --git a/docs/modules/ROOT/pages/Scala_Builtin_Commands.adoc b/docs/modules/ROOT/pages/Scala_Builtin_Commands.adoc index c21d20dc4e0..26593f77878 100644 --- a/docs/modules/ROOT/pages/Scala_Builtin_Commands.adoc +++ b/docs/modules/ROOT/pages/Scala_Builtin_Commands.adoc @@ -4,22 +4,3 @@ include::example/scalalib/basic/4-builtin-commands.adoc[] - -== init - -[source,bash] ----- -> mill -i init com-lihaoyi/mill-scala-hello.g8 -.... -A minimal Scala project. - -name [Scala Seed Project]: hello - -Template applied in ./hello ----- - -The `init` command generates a project based on a Giter8 template. -It prompts you to enter project name and creates a folder with that name. -You can use it to quickly generate a starter project. -There are lots of templates out there for many frameworks and tools! - diff --git a/example/scalalib/basic/4-builtin-commands/build.mill b/example/scalalib/basic/4-builtin-commands/build.mill index 9f284104fbc..0142131156d 100644 --- a/example/scalalib/basic/4-builtin-commands/build.mill +++ b/example/scalalib/basic/4-builtin-commands/build.mill @@ -369,3 +369,40 @@ foo.compileClasspath // Come by our https://discord.gg/MNAXQMAr[Discord Channel] // if you want to ask questions or say hi! // +// +// == init + +/** Usage +> mill init +Run `mill init ` with one of these examples as an argument to download and extract example. +Run `mill init --show-all` to see full list of examples. +Run `mill init ` to generate project from Giter8 template. +... +scalalib/basic/1-simple +... +scalalib/web/1-todo-webapp +scalalib/web/2-webapp-cache-busting +scalalib/web/3-scalajs-module +scalalib/web/4-webapp-scalajs +scalalib/web/5-webapp-scalajs-shared +... +javalib/basic/1-simple +... +javalib/builds/4-realistic +... +javalib/web/1-hello-jetty +javalib/web/2-hello-spring-boot +javalib/web/3-todo-spring-boot +javalib/web/4-hello-micronaut +javalib/web/5-todo-micronaut +kotlinlib/basic/1-simple +... +kotlinlib/builds/4-realistic +... +kotlinlib/web/1-hello-ktor +*/ + +// The `init` command generates a project based on a Mill example project or +// a Giter8 template. You can use it to quickly generate a starter project. +// There are lots of templates out there for many frameworks and tools! + diff --git a/integration/feature/init/src/MillInitTests.scala b/integration/feature/init/src/MillInitTests.scala index 0cf7daa6d98..8b19cb6e67f 100644 --- a/integration/feature/init/src/MillInitTests.scala +++ b/integration/feature/init/src/MillInitTests.scala @@ -7,6 +7,22 @@ object MillInitTests extends UtestIntegrationTestSuite { def tests: Tests = Tests { test("Mill init works") - integrationTest { tester => + import tester._ + val msg = + """Run `mill init ` with one of these examples as an argument to download and extract example. + |Run `mill init --show-all` to see full list of examples. + |Run `mill init ` to generate project from Giter8 template.""".stripMargin + val res = eval("init") + res.isSuccess ==> true + + val exampleListOut = out("init") + val parsed = exampleListOut.json.arr.map(_.str) + assert(parsed.nonEmpty) + assert(res.out.startsWith(msg)) + assert(res.out.endsWith(msg)) + } + + test("Mill init works for g8 templates") - integrationTest { tester => import tester._ eval(("init", "com-lihaoyi/mill-scala-hello.g8", "--name=example")).isSuccess ==> true val projFile = workspacePath / "example/build.sc" diff --git a/kotlinlib/src/mill/kotlinlib/contrib/kover/KoverModule.scala b/kotlinlib/src/mill/kotlinlib/contrib/kover/KoverModule.scala index 64182498ecb..1d7fc0b064c 100644 --- a/kotlinlib/src/mill/kotlinlib/contrib/kover/KoverModule.scala +++ b/kotlinlib/src/mill/kotlinlib/contrib/kover/KoverModule.scala @@ -59,10 +59,11 @@ trait KoverModule extends KotlinModule { outer => Success[String](T.env.getOrElse("KOVER_VERSION", Versions.koverVersion)) } - def koverBinaryReport: T[PathRef] = - Task.Persistent { PathRef(koverDataDir().path / "kover-report.ic") } + def koverBinaryReport: T[PathRef] = Task(persistent = true) { + PathRef(koverDataDir().path / "kover-report.ic") + } - def koverDataDir: T[PathRef] = Task.Persistent { PathRef(T.dest) } + def koverDataDir: T[PathRef] = Task(persistent = true) { PathRef(T.dest) } object kover extends Module with KoverReportBaseModule { diff --git a/main/codesig/package.mill b/main/codesig/package.mill new file mode 100644 index 00000000000..505fd62aaa2 --- /dev/null +++ b/main/codesig/package.mill @@ -0,0 +1,55 @@ +package build.main.codesig +import mill._, scalalib._ + +object `package` extends RootModule with build.MillPublishScalaModule { + override def ivyDeps = Agg(build.Deps.asmTree, build.Deps.osLib, build.Deps.pprint) + def moduleDeps = Seq(build.main.util) + + override lazy val test: CodeSigTests = new CodeSigTests {} + trait CodeSigTests extends MillScalaTests { + val caseKeys = build.interp.watchValue( + os.walk(millSourcePath / "cases", maxDepth = 3) + .map(_.subRelativeTo(millSourcePath / "cases").segments) + .collect { case Seq(a, b, c) => s"$a-$b-$c" } + ) + + def testLogFolder = T { T.dest } + + def caseEnvs[V](f1: CaseModule => Task[V])(s: String, f2: V => String) = { + T.traverse(caseKeys) { i => f1(cases(i)).map(v => s"MILL_TEST_${s}_$i" -> f2(v)) } + } + def forkEnv = T { + Map("MILL_TEST_LOGS" -> testLogFolder().toString) ++ + caseEnvs(_.compile)("CLASSES", _.classes.path.toString)() ++ + caseEnvs(_.compileClasspath)("CLASSPATH", _.map(_.path).mkString(","))() ++ + caseEnvs(_.sources)("SOURCES", _.head.path.toString)() + } + + object cases extends Cross[CaseModule](caseKeys) + trait CaseModule extends ScalaModule with Cross.Module[String] { + def caseName = crossValue + object external extends ScalaModule { + def scalaVersion = build.Deps.scalaVersion + } + + def moduleDeps = Seq(external) + + val Array(prefix, suffix, rest) = caseName.split("-", 3) + def millSourcePath = super.millSourcePath / prefix / suffix / rest + def scalaVersion = build.Deps.scalaVersion + def ivyDeps = T { + if (!caseName.contains("realistic") && !caseName.contains("sourcecode")) super.ivyDeps() + else Agg( + build.Deps.fastparse, + build.Deps.scalatags, + build.Deps.cask, + build.Deps.castor, + build.Deps.mainargs, + build.Deps.requests, + build.Deps.osLib, + build.Deps.upickle + ) + } + } + } +} diff --git a/main/init/package.mill b/main/init/package.mill new file mode 100644 index 00000000000..fea8ebd4fdc --- /dev/null +++ b/main/init/package.mill @@ -0,0 +1,67 @@ +package build.main.init + +import mill._ +import scala.util.matching.Regex + +object `package` extends RootModule with build.MillPublishScalaModule { + def moduleDeps = Seq(build.main) + + override def resources = T.sources { + super.resources() ++ Seq(exampleList()) + } + + def exampleList: T[PathRef] = T { + + val versionPattern: Regex = "\\d+\\.\\d+\\.\\d+".r + val rcPattern: Regex = "-RC\\d+".r + + val millVer: String = build.millVersion() + val lastTag = versionPattern.findFirstMatchIn(millVer) match { + case Some(verMatch) => + val maybeRC = rcPattern.findFirstIn(verMatch.after) + verMatch.matched + maybeRC.getOrElse("") + case None => "0.0.0" + } + + val data: Seq[(os.SubPath, String)] = build.examplePathsWithArtifactName().map { case (path, str) => + val downloadUrl = build.Settings.projectUrl + "/releases/download/" + lastTag + "/" + str + ".zip" + val subPath = path.subRelativeTo(T.workspace / "example") + (subPath, downloadUrl) + } + + val libsSortOrder = List("scalalib", "javalib", "kotlinlib", "extending", "external", "thirdparty", "depth") + val categoriesSortOrder = List("basic", "builds", "web") + + def sortCriterium(strOpt: Option[String], sortOrderList: List[String]): Int = + strOpt.flatMap { str => + val idx = sortOrderList.indexOf(str) + Option.when(idx >= 0)(idx) + }.getOrElse(Int.MaxValue) + + val sortedData = data.sortBy { case (p1, _) => + val segmentsReversed = p1.segments.reverse.lift + val libOpt = segmentsReversed(2) + val categoryOpt = segmentsReversed(1) + val nameOpt = segmentsReversed(0) + + val libSortCriterium = sortCriterium(libOpt, libsSortOrder) + val categorySortCriterium = sortCriterium(categoryOpt, categoriesSortOrder) + val nameSortCriterium = nameOpt.flatMap(_.takeWhile(_.isDigit).toIntOption).getOrElse(Int.MinValue) + (libSortCriterium, libOpt, categorySortCriterium, categoryOpt, nameSortCriterium, nameOpt) + } + + val stream = os.write.outputStream(T.dest / "exampleList.txt") + val writer = new java.io.OutputStreamWriter(stream) + try { + val json = upickle.default.writeTo(sortedData.map { case (p, s) => (p.toString(), s) }, writer) + PathRef(T.dest) + } catch { + case ex: Throwable => T.log.error(ex.toString); throw ex + } finally { + writer.close() + stream.close() + } + + } + +} diff --git a/main/init/src/mill/init/InitModule.scala b/main/init/src/mill/init/InitModule.scala new file mode 100644 index 00000000000..e34b5012818 --- /dev/null +++ b/main/init/src/mill/init/InitModule.scala @@ -0,0 +1,99 @@ +package mill.init + +import mainargs.{Flag, arg} +import mill.api.IO +import mill.define.{Discover, ExternalModule} +import mill.util.Util.download +import mill.{Command, Module, T} + +import java.io.IOException +import java.util.UUID +import scala.util.{Failure, Success, Try, Using} + +object InitModule extends ExternalModule with InitModule { + lazy val millDiscover: Discover = Discover[this.type] +} + +trait InitModule extends Module { + + type ExampleUrl = String + type ExampleId = String + + val msg: String = + """Run `mill init ` with one of these examples as an argument to download and extract example. + |Run `mill init --show-all` to see full list of examples. + |Run `mill init ` to generate project from Giter8 template.""".stripMargin + def moduleNotExistMsg(id: String): String = s"Example [$id] is not present in examples list" + def directoryExistsMsg(extractionTargetPath: String): String = + s"Can't download example, because extraction directory [$extractionTargetPath] already exist" + + /** + * @return Seq of example names or Seq with path to parent dir where downloaded example was unpacked + */ + def init( + @mainargs.arg(positional = true, short = 'e') exampleId: Option[ExampleId], + @arg(name = "show-all") showAll: Flag = Flag() + ): Command[Seq[String]] = + T.command { + usingExamples { examples => + val result: Try[(Seq[String], String)] = exampleId match { + case None => + val exampleIds: Seq[ExampleId] = examples.map { case (exampleId, _) => exampleId } + def fullMessage(exampleIds: Seq[ExampleId]) = + msg + "\n\n" + exampleIds.mkString("\n") + "\n\n" + msg + if (showAll.value) + Success((exampleIds, fullMessage(exampleIds))) + else { + val toShow = List("basic", "builds", "web") + val filteredIds = + exampleIds.filter(_.split('/').lift.apply(1).exists(toShow.contains)) + Success((filteredIds, fullMessage(filteredIds))) + } + case Some(value) => + val result: Try[(Seq[String], String)] = for { + url <- examples.toMap.get(value).toRight(new Exception( + moduleNotExistMsg(value) + )).toTry + extractedDirName = { + val zipName = url.split('/').last + if (zipName.toLowerCase.endsWith(".zip")) zipName.dropRight(4) else zipName + } + downloadDir = T.workspace + downloadPath = downloadDir / extractedDirName + _ <- if (os.exists(downloadPath)) Failure(new IOException( + directoryExistsMsg(downloadPath.toString) + )) + else Success(()) + path <- + Try({ + val tmpName = UUID.randomUUID().toString + ".zip" + val downloaded = download(url, os.rel / tmpName)(downloadDir) + val unpacked = IO.unpackZip(downloaded.path, os.rel)(downloadDir) + Try(os.remove(downloaded.path)) + unpacked + }).recoverWith(ex => + Failure( + new IOException(s"Couldn't download example: [$value];\n ${ex.getMessage}") + ) + ) + } yield (Seq(path.path.toString()), s"Example downloaded to [$downloadPath]") + + result + } + result + }.flatten match { + case Success((ret, msg)) => + T.log.outputStream.println(msg) + ret + case Failure(exception) => + T.log.error(exception.getMessage) + throw exception + } + } + private def usingExamples[T](fun: Seq[(ExampleId, ExampleUrl)] => T): Try[T] = + Using(getClass.getClassLoader.getResourceAsStream("exampleList.txt")) { exampleList => + val reader = upickle.default.reader[Seq[(ExampleId, ExampleUrl)]] + val exampleNames: Seq[(ExampleId, ExampleUrl)] = upickle.default.read(exampleList)(reader) + fun(exampleNames) + } +} diff --git a/main/init/test/src/mill/init/InitModuleTests.scala b/main/init/test/src/mill/init/InitModuleTests.scala new file mode 100644 index 00000000000..2a6c854d872 --- /dev/null +++ b/main/init/test/src/mill/init/InitModuleTests.scala @@ -0,0 +1,72 @@ +package mill.init + +import mill.api.{PathRef, Result, Val} +import mill.{Agg, T} +import mill.define.{Cross, Discover, Module, Task} +import mill.testkit.UnitTester +import mill.testkit.TestBaseModule +import mill.testkit.UnitTester +import mill.testkit.TestBaseModule +import utest._ + +import java.io.{ByteArrayOutputStream, PrintStream} +import scala.util.Using + +object InitModuleTests extends TestSuite { + + override def tests: Tests = Tests { + + test("init") { + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + object initmodule extends TestBaseModule with InitModule + val evaluator = UnitTester( + initmodule, + null, + outStream = new PrintStream(outStream, true), + errStream = new PrintStream(errStream, true) + ) + test("no args") { + val results = evaluator.evaluator.evaluate(Agg(initmodule.init(None))) + + assert(results.failing.keyCount == 0) + + val Result.Success(Val(value)) = results.rawValues.head + val consoleShown = outStream.toString + + val examplesList: Seq[String] = value.asInstanceOf[Seq[String]] + assert( + consoleShown.startsWith(initmodule.msg), + examplesList.forall(_.nonEmpty) + ) + } + test("non existing example") { + val nonExistingModuleId = "nonExistingExampleId" + val results = evaluator.evaluator.evaluate(Agg(initmodule.init(Some(nonExistingModuleId)))) + assert(results.failing.keyCount == 1) + assert(errStream.toString.contains(initmodule.moduleNotExistMsg(nonExistingModuleId))) + } + test("mill init errors if directory already exist") { + type ExampleId = String + type ExampleUrl = String + val examplesList = + Using(initmodule.getClass.getClassLoader.getResourceAsStream("exampleList.txt")) { + examplesSource => + val reader = upickle.default.reader[Seq[(ExampleId, ExampleUrl)]] + val examples: Seq[(ExampleId, ExampleUrl)] = + upickle.default.read(examplesSource)(reader) + examples + }.get + val exampleId = examplesList.head._1 + val targetDir = examplesList.head._2.dropRight(4).split("/").last // dropping .zip + + val targetPath = initmodule.millSourcePath / targetDir + os.makeDir(targetPath) + val results = evaluator.evaluator.evaluate(Agg(initmodule.init(Some(exampleId)))) + assert(results.failing.keyCount == 1) + assert(errStream.toString.contains(initmodule.directoryExistsMsg(targetPath.toString()))) + + } + } + } +} diff --git a/main/package.mill b/main/package.mill index 7844a161a9a..2afb57b74e2 100644 --- a/main/package.mill +++ b/main/package.mill @@ -93,59 +93,6 @@ object `package` extends RootModule with build.MillStableScalaModule with BuildI def ivyDeps = Agg(build.Deps.coursier, build.Deps.jline) } - object codesig extends build.MillPublishScalaModule { - override def ivyDeps = Agg(build.Deps.asmTree, build.Deps.osLib, build.Deps.pprint) - def moduleDeps = Seq(util) - - override lazy val test: CodeSigTests = new CodeSigTests {} - trait CodeSigTests extends MillScalaTests { - val caseKeys = build.interp.watchValue( - os.walk(millSourcePath / "cases", maxDepth = 3) - .map(_.subRelativeTo(millSourcePath / "cases").segments) - .collect { case Seq(a, b, c) => s"$a-$b-$c" } - ) - - def testLogFolder = T { T.dest } - - def caseEnvs[V](f1: CaseModule => Task[V])(s: String, f2: V => String) = { - T.traverse(caseKeys) { i => f1(cases(i)).map(v => s"MILL_TEST_${s}_$i" -> f2(v)) } - } - def forkEnv = T { - Map("MILL_TEST_LOGS" -> testLogFolder().toString) ++ - caseEnvs(_.compile)("CLASSES", _.classes.path.toString)() ++ - caseEnvs(_.compileClasspath)("CLASSPATH", _.map(_.path).mkString(","))() ++ - caseEnvs(_.sources)("SOURCES", _.head.path.toString)() - } - - object cases extends Cross[CaseModule](caseKeys) - trait CaseModule extends ScalaModule with Cross.Module[String] { - def caseName = crossValue - object external extends ScalaModule { - def scalaVersion = build.Deps.scalaVersion - } - - def moduleDeps = Seq(external) - - val Array(prefix, suffix, rest) = caseName.split("-", 3) - def millSourcePath = super.millSourcePath / prefix / suffix / rest - def scalaVersion = build.Deps.scalaVersion - def ivyDeps = T { - if (!caseName.contains("realistic") && !caseName.contains("sourcecode")) super.ivyDeps() - else Agg( - build.Deps.fastparse, - build.Deps.scalatags, - build.Deps.cask, - build.Deps.castor, - build.Deps.mainargs, - build.Deps.requests, - build.Deps.osLib, - build.Deps.upickle - ) - } - } - } - } - object define extends build.MillStableScalaModule { def moduleDeps = Seq(api, util) def compileIvyDeps = Agg(build.Deps.scalaReflect(scalaVersion())) diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index aead83caada..799a53fa894 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -496,20 +496,37 @@ trait MainModule extends BaseModule0 { } /** - * The `init` command generates a project based on a Giter8 template. It - * prompts you to enter project name and creates a folder with that name. - * You can use it to quickly generate a starter project. There are lots of - * templates out there for many frameworks and tools! + * The `init` allows you to quickly generate a starter project. + * + * If you run it without arguments, it displays the list of available examples. + * + * If you pass one of listed examples, it downloads specified example from mill releases page and extracts it to working directory. + * + * If you pass a g8 template, it will generate a project based on a Giter8 template. + * It prompts you to enter project name and creates a folder with that name. + * There are lots of templates out there for many frameworks and tools! */ - def init(evaluator: Evaluator, args: String*): Command[Unit] = Task.Command(exclusive = true) { - RunScript.evaluateTasksNamed( - evaluator, - Seq("mill.scalalib.giter8.Giter8Module/init") ++ args, - SelectMode.Separated - ) - - () - } + def init(evaluator: Evaluator, args: String*): Command[ujson.Value] = + Task.Command(exclusive = true) { + val evaluated = + if (args.headOption.exists(_.toLowerCase.endsWith(".g8"))) + RunScript.evaluateTasksNamed( + evaluator, + Seq("mill.scalalib.giter8.Giter8Module/init") ++ args, + SelectMode.Separated + ) + else + RunScript.evaluateTasksNamed( + evaluator, + Seq("mill.init.InitModule/init") ++ args, + SelectMode.Separated + ) + evaluated match { + case Left(failStr) => throw new Exception(failStr) + case Right((_, Right(Seq((_, Some((_, jsonableResult))))))) => jsonableResult + case Right((_, Left(failStr))) => throw new Exception(failStr) + } + } private type VizWorker = ( LinkedBlockingQueue[(scala.Seq[_], scala.Seq[_], os.Path)], diff --git a/main/util/src/mill/util/LinePrefixOutputStream.scala b/main/util/src/mill/util/LinePrefixOutputStream.scala index 3145b9ba28e..5a83c38c70f 100644 --- a/main/util/src/mill/util/LinePrefixOutputStream.scala +++ b/main/util/src/mill/util/LinePrefixOutputStream.scala @@ -46,8 +46,9 @@ class LinePrefixOutputStream( isNewLine = true start = i writeOutBuffer() + } else { + i += 1 } - i += 1 } if (math.min(i, max) - start > 0) { diff --git a/main/util/test/src/mill/util/LinePrefixOutputStreamTests.scala b/main/util/test/src/mill/util/LinePrefixOutputStreamTests.scala index ca114de48dd..eba2614e337 100644 --- a/main/util/test/src/mill/util/LinePrefixOutputStreamTests.scala +++ b/main/util/test/src/mill/util/LinePrefixOutputStreamTests.scala @@ -42,6 +42,17 @@ object LinePrefixOutputStreamTests extends TestSuite { assert(baos.toString == "PREFIXhello\nPREFIXworld\n") } + test("allAtOnceDoubleNewline") { + val baos = new ByteArrayOutputStream() + val lpos = new LinePrefixOutputStream("PREFIX", baos) + val arr = "hello\n\nworld\n\n".getBytes() + lpos.write(arr) + lpos.flush() + + val expected = "PREFIXhello\nPREFIX\nPREFIXworld\nPREFIX\n" + assert(baos.toString == expected) + } + test("ranges") { for (str <- Seq("hello\nworld\n")) { val arr = str.getBytes()