diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 03a0db2..39bc41c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,7 +16,7 @@ on: types: [published] jobs: - bake: + bake_no_cache: runs-on: ubuntu-latest steps: - name: Checkout @@ -34,6 +34,28 @@ jobs: workdir: ros2_pkg_builder/docker files: | ./docker-bake.hcl + targets: | + builder + bake_cache_for_robotx_buildfarm: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and push + uses: docker/bake-action@v3 + with: + push: true + workdir: ros2_pkg_builder/docker + files: | + ./docker-bake.hcl + targets: | + cache build_python_packages: strategy: fail-fast: false diff --git a/ros2_pkg_builder/build.py b/ros2_pkg_builder/build.py index df4a790..9166981 100644 --- a/ros2_pkg_builder/build.py +++ b/ros2_pkg_builder/build.py @@ -25,6 +25,7 @@ def build_deb_packages( build_builder_image: bool, packages_above: str, apt_server: str, + cache_image: str, ) -> None: logger = get_logger(__name__) logger.info("Start building debian packages.") @@ -45,6 +46,19 @@ def build_deb_packages( .joinpath("docker") .joinpath("workspace.repos"), ) + if cache_image == "": + logger.info( + "Cache image was not specified, using ros:" + rosdistro + " as base image." + ) + cache_image = "ros" + else: + logger.info( + "Cache image was specified, using " + + cache_image + + ":" + + rosdistro + + " as base image." + ) if build_builder_image: logger.info("Building builder image for " + architecture + "/" + rosdistro) docker.buildx.bake( @@ -53,6 +67,7 @@ def build_deb_packages( set={ "*.platform": "linux/" + architecture, "*.context": Path(ros2_pkg_builder.__path__[0]).joinpath("docker"), + "*.args.IMAGE_NAME": cache_image, }, files=[ Path(ros2_pkg_builder.__path__[0]) @@ -60,6 +75,7 @@ def build_deb_packages( .joinpath("docker-bake.hcl") ], ) + docker.run( image="wamvtan/ros2_pkg_builder:" + rosdistro, volumes=[ @@ -105,6 +121,14 @@ def main(): default="", help="List of build target packages.", ) + parser.add_argument( + "--cache-image", + type=str, + default="", + help="Docker image for caching rosdep dependency. \ + This image is used for base image of builder image. \ + If this value is empty, it means use no cache.", + ) parser.add_argument("--apt-server", type=str, default="ftp.jaist.ac.jp/pub/Linux") args = parser.parse_args() build_deb_packages( @@ -114,6 +138,7 @@ def main(): args.build_builder_image, args.packages_above, args.apt_server, + args.cache_image, ) diff --git a/ros2_pkg_builder/docker/Dockerfile.builder b/ros2_pkg_builder/docker/Dockerfile.builder index 7e24f5f..b200f55 100644 --- a/ros2_pkg_builder/docker/Dockerfile.builder +++ b/ros2_pkg_builder/docker/Dockerfile.builder @@ -1,5 +1,6 @@ ARG ROS_DISTRO=humble -FROM ros:${ROS_DISTRO} as build-base-stage +ARG IMAGE_NAME=ros +FROM ${IMAGE_NAME}:${ROS_DISTRO} as build-stage RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive \ apt-get install -y \ diff --git a/ros2_pkg_builder/docker/Dockerfile.cache b/ros2_pkg_builder/docker/Dockerfile.cache new file mode 100644 index 0000000..f6b1f84 --- /dev/null +++ b/ros2_pkg_builder/docker/Dockerfile.cache @@ -0,0 +1,16 @@ +ARG ROS_DISTRO=humble +ARG IMAGE_NAME=ros +FROM ${IMAGE_NAME}:${ROS_DISTRO} as cache-stage + +WORKDIR /workspace +RUN echo "repositories:" >> workspace.repos + +ARG REPOS_FILE=workspace.repos +ADD ${REPOS_FILE} workspace.repos + +ARG APT_SERVER=ftp.jaist.ac.jp/pub/Linux +RUN sed -i "s@archive.ubuntu.com@$APT_SERVER@g" /etc/apt/sources.list +RUN vcs import . < workspace.repos +RUN apt update && rosdep install -iy --from-paths . --skip-keys $(colcon list -n | tr '\n' ',') && \ + apt-get clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/ros2_pkg_builder/docker/docker-bake.hcl b/ros2_pkg_builder/docker/docker-bake.hcl index 613c880..3574948 100644 --- a/ros2_pkg_builder/docker/docker-bake.hcl +++ b/ros2_pkg_builder/docker/docker-bake.hcl @@ -1,9 +1,9 @@ -group "default" { +group "builder" { targets = ["latest", "iron", "humble", "rolling"] } target "latest" { - target = "build-base-stage" + target = "build-stage" dockerfile = "Dockerfile.builder" tags = ["docker.io/wamvtan/ros2_pkg_builder:latest"] args = { @@ -35,3 +35,43 @@ target "humble" { "ROS_DISTRO" : "humble" } } + +// Grid map does not released in comment outed distribution +group "cache" { + targets = ["cache-latest", "cache-humble"] # "cache-iron", "cache-rolling" ] +} + +target "cache-latest" { + target = "cache-stage" + dockerfile = "Dockerfile.cache" + tags = ["docker.io/wamvtan/robotx_buildfarm:latest"] + args = { + "ROS_DISTRO" : "latest", + "REPOS_FILE" : "https://raw.githubusercontent.com/OUXT-Polaris/ros2_pkg_builder/ouxt/repos/workspace.repos" + } + platforms = ["linux/amd64", "linux/arm64/v8"] +} + +target "cache-rolling" { + inherits = ["cache-latest"] + tags = ["docker.io/wamvtan/robotx_buildfarm:rolling"] + args = { + "ROS_DISTRO" : "rolling" + } +} + +target "cache-iron" { + inherits = ["cache-latest"] + tags = ["docker.io/wamvtan/robotx_buildfarm:iron"] + args = { + "ROS_DISTRO" : "iron" + } +} + +target "cache-humble" { + inherits = ["cache-latest"] + tags = ["docker.io/wamvtan/robotx_buildfarm:humble"] + args = { + "ROS_DISTRO" : "humble" + } +} diff --git a/ros2_pkg_builder/docker/entrypoint.sh b/ros2_pkg_builder/docker/entrypoint.sh index 80a3368..d967491 100644 --- a/ros2_pkg_builder/docker/entrypoint.sh +++ b/ros2_pkg_builder/docker/entrypoint.sh @@ -1,24 +1,25 @@ #!/bin/bash - -# Building packages echo "Changing apt server to $APT_SERVER" sed -i "s@archive.ubuntu.com@$APT_SERVER@g" /etc/apt/sources.list -vcs import . < workspace.repos +cd /artifacts +sh update_apt_repo.sh -apt update -rosdep install -iy -t buildtool_export -t build -t buildtool -t build_export -t test --from-paths . --skip-keys $(colcon list -n | tr '\n' ',') +cd /workspace + +vcs import . < workspace.repos colcon list -t -p | xargs -L 1 bash -c \ 'cd "$1"; pkg=$(colcon list -n);pkg_with_prefix=$(echo $pkg|echo "ros-humble-"$(sed -e 's/_/-/g')); \ yq eval -i ".\"$pkg\".ubuntu=[\"$pkg_with_prefix\"]" /artifacts/rosdep/${ROS_DISTRO}.yaml' _ rosdep update -cd /artifacts && sh update_apt_repo.sh +apt update +rosdep install -iy --from-paths . --skip-keys $(colcon list -n | tr '\n' ',') cd /workspace && colcon list -t -p --packages-above $PACKAGES_ABOVE | xargs -I{} bash -c \ 'echo {} && cd {} && \ - rosdep install -iy -t buildtool_export -t build -t buildtool -t build_export -t test --from-paths . && \ + rosdep install -iy --from-paths . && \ bloom-generate rosdebian --os-name ubuntu --os-version $DEB_DISTRO --ros-distro $ROS_DISTRO && \ fakeroot debian/rules binary && cd /artifacts && sh update_apt_repo.sh && \ dpkg -i /artifacts/dists/$DEB_DISTRO/universe/binary-$ARCHITECTURE/ros-$ROS_DISTRO-*.deb'