Merge pull request #108 from ecpullen/alpha-build

twoliter: `twoliter build` alpha

twoliter/embedded/Dockerfile

@@ -144,7 +144,6 @@ USER builder
 RUN --mount=source=.cargo,target=/home/builder/.cargo \
     --mount=type=cache,target=/home/builder/.cache,from=cache,source=/cache \
     --mount=type=cache,target=/home/builder/rpmbuild/BUILD/sources/models/src/variant,from=variantcache,source=/variantcache \
-    --mount=type=cache,target=/home/builder/rpmbuild/BUILD/sources/logdog/conf/current,from=variantcache,source=/variantcache \
     --mount=source=sources,target=/home/builder/rpmbuild/BUILD/sources \
     rpmbuild -ba --clean \
       --undefine _auto_set_build_flags \

twoliter/embedded/Makefile.toml

@@ -97,7 +97,8 @@ BUILDSYS_JOBS = "8"
 CARGO_HOME = "${BUILDSYS_ROOT_DIR}/.cargo"
 # This needs to end with pkg/mod so that we can mount the parent of pkg/mod as GOPATH.
 GO_MOD_CACHE = "${BUILDSYS_ROOT_DIR}/.gomodcache/pkg/mod"
-GO_MODULES = "ecs-gpu-init host-ctr"
+# Dynamically load a list of go modules from ${BUILDSYS_SOURCE_DIR}
+GO_MODULES = { script = ['find ${BUILDSYS_SOURCES_DIR} -name go.mod -type f -printf "%h\n" | xargs -n1 basename'] }
 DOCKER_BUILDKIT = "1"
 
 # This is the filename suffix for operations that write out AMI information to

twoliter/embedded/rpm2img

@@ -238,7 +238,6 @@ INVENTORY_DATA="$(jq --slurp 'sort_by(.Name)' <<< "${INVENTORY_DATA}" | jq '{"Co
 printf "%s\n" "${INVENTORY_DATA}" > "${ROOT_MOUNT}/usr/share/bottlerocket/application-inventory.json"
 
 # install licenses
-install -p -m 0644 /host/{COPYRIGHT,LICENSE-APACHE,LICENSE-MIT} "${ROOT_MOUNT}"/usr/share/licenses/
 mksquashfs \
   "${ROOT_MOUNT}"/usr/share/licenses \
   "${ROOT_MOUNT}"/usr/share/bottlerocket/licenses.squashfs \

twoliter/src/cargo_make.rs

@@ -94,41 +94,8 @@ impl CargoMake {
         self
     }
 
-    /// Specify environment variables that should be applied for this comand
-    pub(crate) fn _envs<S1, S2, V>(mut self, env_vars: V) -> Self
-    where
-        S1: Into<String>,
-        S2: Into<String>,
-        V: Into<Vec<(S1, S2)>>,
-    {
-        for (key, value) in env_vars.into() {
-            self.args
-                .push(format!("-e={}={}", key.into(), value.into()));
-        }
-        self
-    }
-
-    /// Specify `cargo make` arguments that should be applied for this comand
-    pub(crate) fn _arg<S>(mut self, arg: S) -> Self
-    where
-        S: Into<String>,
-    {
-        self.args.push(arg.into());
-        self
-    }
-
-    /// Specify `cargo make` arguments that should be applied for this comand
-    pub(crate) fn _args<V, S>(mut self, args: V) -> Self
-    where
-        S: Into<String>,
-        V: Into<Vec<S>>,
-    {
-        self.args.extend(args.into().into_iter().map(Into::into));
-        self
-    }
-
     /// Execute the `cargo make` task
-    pub(crate) async fn _exec<S>(&self, task: S) -> Result<()>
+    pub(crate) async fn exec<S>(&self, task: S) -> Result<()>
     where
         S: Into<String>,
     {

twoliter/src/cmd/build.rs

@@ -1,6 +1,14 @@
-use anyhow::Result;
+use crate::cargo_make::CargoMake;
+use crate::docker::DockerContainer;
+use crate::project;
+use crate::tools::{install_tools, tools_tempdir};
+use anyhow::{Context, Result};
 use clap::Parser;
-use std::path::PathBuf;
+use log::debug;
+use std::fs;
+use std::path::{Path, PathBuf};
+use tempfile::TempDir;
+use tokio::fs::{remove_dir_all, remove_file};
 
 #[derive(Debug, Parser)]
 pub(crate) enum BuildCommand {
@@ -25,10 +33,95 @@ pub(crate) struct BuildVariant {
     /// The architecture to build for.
     #[clap(long = "arch", default_value = "x86_64")]
     arch: String,
+
+    /// The variant to build.
+    variant: String,
 }
 
 impl BuildVariant {
     pub(super) async fn run(&self) -> Result<()> {
-        Ok(())
+        let project = project::load_or_find_project(self.project_path.clone()).await?;
+        let token = project.token();
+        let tempdir = tools_tempdir()?;
+        install_tools(&tempdir).await?;
+        let makefile_path = tempdir.path().join("Makefile.toml");
+        // A temporary directory in the `build` directory
+        let build_temp_dir = TempDir::new_in(project.project_dir())
+            .context("Unable to create a tempdir for Twoliter's build")?;
+        let packages_dir = build_temp_dir.path().join("sdk_rpms");
+        fs::create_dir_all(&packages_dir)?;
+
+        let sdk_container = DockerContainer::new(
+            format!("sdk-{}", token),
+            project
+                .sdk(&self.arch)
+                .context(format!(
+                    "No SDK defined in {} for {}",
+                    project.filepath().display(),
+                    &self.arch
+                ))?
+                .uri(),
+        )
+        .await?;
+        sdk_container
+            .cp_out(Path::new("twoliter/alpha/build/rpms"), &packages_dir)
+            .await?;
+
+        let rpms_dir = project.project_dir().join("build").join("rpms");
+        fs::create_dir_all(&rpms_dir)?;
+        debug!("Moving rpms to build dir");
+        for maybe_file in fs::read_dir(packages_dir.join("rpms"))? {
+            let file = maybe_file?;
+            debug!("Moving '{}'", file.path().display());
+            fs::rename(file.path(), rpms_dir.join(file.file_name()))?;
+        }
+
+        let mut created_files = Vec::new();
+
+        let sbkeys_dir = project.project_dir().join("sbkeys");
+        if !sbkeys_dir.is_dir() {
+            // Create a sbkeys directory in the main project
+            debug!("sbkeys dir not found. Creating a temporary directory");
+            fs::create_dir_all(&sbkeys_dir)?;
+            sdk_container
+                .cp_out(
+                    Path::new("twoliter/alpha/sbkeys/generate-local-sbkeys"),
+                    &sbkeys_dir,
+                )
+                .await?;
+        };
+
+        // TODO: Remove once models is no longer conditionally compiled.
+        // Create the models directory for the sdk to mount
+        let models_dir = project.project_dir().join("sources/models");
+        if !models_dir.is_dir() {
+            debug!("models source dir not found. Creating a temporary directory");
+            fs::create_dir_all(&models_dir.join("src/variant"))
+                .context("Unable to create models source directory")?;
+            created_files.push(models_dir)
+        }
+
+        // Hold the result of the cargo make call so we can clean up the project directory first.
+        let res = CargoMake::new(&project, &self.arch)?
+            .env("TWOLITER_TOOLS_DIR", tempdir.path().display().to_string())
+            .env("BUILDSYS_ARCH", &self.arch)
+            .env("BUILDSYS_VARIANT", &self.variant)
+            .env("BUILDSYS_SBKEYS_DIR", sbkeys_dir.display().to_string())
+            .makefile(makefile_path)
+            .project_dir(project.project_dir())
+            .exec("build")
+            .await;
+
+        // Clean up all of the files we created
+        for file_name in created_files {
+            let added = Path::new(&file_name);
+            if added.is_file() {
+                remove_file(added).await?;
+            } else if added.is_dir() {
+                remove_dir_all(added).await?;
+            }
+        }
+
+        res
     }
 }

twoliter/src/docker/container.rs

@@ -0,0 +1,74 @@
+use crate::common::exec;
+use anyhow::Result;
+use log::{debug, log, Level};
+use std::path::Path;
+use tokio::process::Command;
+
+pub(crate) struct DockerContainer {
+    name: String,
+}
+
+impl DockerContainer {
+    /// Create a docker image with the given name from the image by using `docker create`.
+    pub(crate) async fn new<S1, S2>(container_name: S1, image: S2) -> Result<Self>
+    where
+        S1: Into<String>,
+        S2: Into<String>,
+    {
+        let name = container_name.into();
+        let image = image.into();
+
+        // Make sure previous versions of this container are stopped deleted.
+        cleanup_container(&name, Level::Trace).await;
+
+        debug!("Creating docker container '{name}' from image '{image}'");
+
+        // Create the new container.
+        let args = vec![
+            "create".to_string(),
+            "--rm".to_string(),
+            "--name".to_string(),
+            name.to_string(),
+            image.to_string(),
+        ];
+
+        exec(Command::new("docker").args(args), true).await?;
+        Ok(Self { name })
+    }
+
+    /// Copy the data from this container to a local destination.
+    pub(crate) async fn cp_out<P1, P2>(&self, src: P1, dest: P2) -> Result<()>
+    where
+        P1: AsRef<Path>,
+        P2: AsRef<Path>,
+    {
+        debug!(
+            "Copying '{}' from '{}' to '{}'",
+            src.as_ref().display(),
+            self.name,
+            dest.as_ref().display()
+        );
+        let mut args = vec!["cp".to_string()];
+        args.push(format!("{}:{}", self.name, src.as_ref().display()));
+        args.push(dest.as_ref().display().to_string());
+        exec(Command::new("docker").args(args), true).await
+    }
+}
+
+impl Drop for DockerContainer {
+    fn drop(&mut self) {
+        let name = self.name.clone();
+        tokio::task::spawn(async move { cleanup_container(&name, Level::Error).await });
+    }
+}
+
+async fn cleanup_container(name: &str, log_level: Level) {
+    let args = vec!["stop".to_string(), name.to_string()];
+    if let Err(e) = exec(Command::new("docker").args(args), true).await {
+        log!(log_level, "Unable to stop container '{}': {e}", name)
+    }
+    let args = vec!["rm".to_string(), name.to_string()];
+    if let Err(e) = exec(Command::new("docker").args(args), true).await {
+        log!(log_level, "Unable to remove container '{}': {e}", name)
+    }
+}

twoliter/src/docker/mod.rs

@@ -1,5 +1,7 @@
 mod commands;
+mod container;
 mod image;
 
-#[allow(unused)]
+pub(crate) use self::container::DockerContainer;
+#[allow(unused_imports)]
 pub(crate) use self::image::{ImageArchUri, ImageUri};

twoliter/src/project.rs

@@ -5,6 +5,7 @@ use log::{debug, trace};
 use non_empty_string::NonEmptyString;
 use serde::de::Error;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use sha2::{Digest, Sha512};
 use std::fmt;
 use std::path::{Path, PathBuf};
 use tokio::fs;
@@ -115,6 +116,13 @@ impl Project {
     pub(crate) fn toolchain(&self, arch: &str) -> Option<ImageArchUri> {
         self.toolchain_name().map(|s| s.uri(arch))
     }
+
+    pub(crate) fn token(&self) -> String {
+        let mut d = Sha512::new();
+        d.update(self.filepath().display().to_string());
+        let digest = hex::encode(d.finalize());
+        (digest[..12]).to_string()
+    }
 }
 
 /// A base name for an image that can be suffixed using a naming convention. For example,

twoliter/src/test/cargo_make.rs

@@ -7,7 +7,7 @@ async fn test_cargo_make() {
     let cargo_make = CargoMake::new(&project, "arch")
         .unwrap()
         .makefile(data_dir().join("Makefile.toml"));
-    cargo_make._exec("verify-twoliter-env").await.unwrap();
+    cargo_make.exec("verify-twoliter-env").await.unwrap();
     cargo_make
         .clone()
         .env("FOO", "bar")
@@ -29,23 +29,4 @@ async fn test_cargo_make() {
         )
         .await
         .unwrap();
-    cargo_make
-        .clone()
-        ._arg("--env")
-        ._arg("FOO=bar")
-        .exec_with_args("verify-env-value-with-arg", ["FOO", "bar"])
-        .await
-        .unwrap();
-    cargo_make
-        .clone()
-        ._args(["--env", "FOO=bar"])
-        .exec_with_args("verify-env-value-with-arg", ["FOO", "bar"])
-        .await
-        .unwrap();
-    cargo_make
-        .clone()
-        ._envs([("FOO", "bar"), ("BAR", "baz")])
-        .exec_with_args("verify-env-value-with-arg", ["BAR", "baz"])
-        .await
-        .unwrap();
 }