A whole new builder.

Rewritten in python.
Only replace `build.sh` (but can easily replace scons runners and
`build-release.sh`).
See `./cli.py -h` for help.

Author: Faless

.gitignore

@@ -15,3 +15,10 @@ releases/
 sha512sums/
 tmp/
 web/
+
+# Editors
+*.swp
+
+# Python
+*.pyc
+__pycache__/

builder/__init__.py

@@ -0,0 +1,5 @@
+from .builder import PodmanRunner, GitRunner
+from .images import configs as ImageConfigs
+from .config import Config, write_config, load_config
+
+BuildConfigs = [] # TODO

builder/builder.py

@@ -0,0 +1,163 @@
+#!/usr/python
+
+import logging
+import os
+import subprocess
+import sys
+
+from .config import Config
+from .runner import RunError, Runner, run_simple
+
+
+def which(what):
+    return run_simple(["which", what]).stdout.strip()
+
+
+def ensure_dir(dirname):
+    os.makedirs(dirname, exist_ok=True)
+
+
+class PodmanRunner(Runner):
+    IMAGES = ["mono-glue", "windows", "ubuntu-64", "ubuntu-32", "javascript"]
+    IMAGES_PRIVATE = ["macosx", "android", "ios", "uwp"]
+
+    @staticmethod
+    def get_images():
+        return PodmanRunner.IMAGES + PodmanRunner.IMAGES_PRIVATE
+
+    def get_image_path(self, image, version="latest", local=False):
+        if local:
+            return f"localhost/{image}:{version}"
+        path = Config.private_path if image in PodmanRunner.IMAGES_PRIVATE else Config.public_path
+        return f"{Config.registry}/{path}/{image}:{version}"
+
+    def __init__(self, base_dir, dry_run=False):
+        self.base_dir = base_dir
+        self.dry_run = dry_run
+        self.logged_in = False
+        self._podman = self._detect_podman()
+
+    def _detect_podman(self):
+        podman = which("podman")
+        if not podman:
+            podman = which("docker")
+        if not podman:
+            print("Either podman or docker needs to be installed")
+            sys.exit(1)
+        return podman
+
+    def login(self):
+        if Config.username == "" or Config.password == "":
+            logging.debug("Skipping login, missing username or password")
+            return
+        self.logged_in = run_simple(self._podman, "login", Config.regitry, "-u", Config.username, "-p", Config.password).returncode == 0
+
+    def image_exists(self, image):
+        return run_simple([self._podman, "image", "exists", image]).returncode == 0
+
+    def fetch_image(self, image, force=False):
+        exists = not force and self.image_exists(image)
+        if not exists:
+            self.run([self._podman, "pull", "%s/%s" % (Config.registry, image)])
+
+    def fetch_images(self, images=[], **kwargs):
+        if len(images) == 0:
+            images = PodmanRunner.get_images()
+        for image in images:
+            if image in PodmanRunner.IMAGES:
+                self.fetch_image("%s/%s" % (Config.public_path, image), **kwargs)
+            elif image in PodmanRunner.IMAGES_PRIVATE:
+                if not self.logged_in:
+                    print("Can't fetch image: %s. Not logged in" % image)
+                    continue
+                self.fetch_image("%s/%s" % (Config.private_path, image), **kwargs)
+
+    def podrun(self, run_config, classical=False, mono=False, local=False, interactive=False, **kwargs):
+        def env(env_vars):
+            for k, v in env_vars.items():
+                yield("--env")
+                yield(f"{k}={v}")
+
+        def mount(mount_points):
+            for k, v in mount_points.items():
+                yield("-v")
+                yield(f"{self.base_dir}/{k}:/root/{v}")
+
+        for d in run_config.dirs:
+            ensure_dir(os.path.join(self.base_dir, d))
+
+        cores = os.environ.get('NUM_CORES', os.cpu_count())
+        cmd = [self._podman, "run", "--rm", "-w", "/root/"]
+        cmd += env({
+            "BUILD_NAME": os.environ.get("BUILD_NAME", "custom_build"),
+            "NUM_CORES": os.environ.get("NUM_CORES", os.cpu_count()),
+            "CLASSICAL": 1 if classical else 0,
+            "MONO": 1 if mono else 0,
+        })
+        cmd += mount({
+            "mono-glue": "mono-glue",
+            "godot.tar.gz": "godot.tar.gz",
+        })
+        cmd += mount(run_config.mounts)
+        if run_config.out_dir is not None:
+            out_dir = f"out/{run_config.out_dir}"
+            ensure_dir(f"{self.base_dir}/{out_dir}")
+            cmd += mount({
+                out_dir: "out"
+            })
+        cmd += run_config.extra_opts
+
+        image_path = self.get_image_path(run_config.image, version=run_config.image_version, local=local)
+        if interactive:
+            if self.dry_run:
+                print(" ".join(cmd + ["-it", image_path, "bash"]))
+                return
+            return subprocess.run(cmd + ["-it", image_path, "bash"])
+
+        cmd += [image_path] + run_config.cmd
+        if run_config.log and not 'log' in kwargs:
+            ensure_dir(f"{self.base_dir}/out/logs")
+            with open(os.path.join(self.base_dir, "out", "logs", run_config.log), "w") as log:
+                return self.run(cmd, log=log, **kwargs)
+        else:
+            return self.run(cmd, **kwargs)
+
+
+class GitRunner(Runner):
+
+    def __init__(self, base_dir, dry_run=False):
+        self.dry_run = dry_run
+        self.base_dir = base_dir
+
+    def git(self, *args, can_fail=False):
+        return self.run(["git"] + list(args), can_fail)
+
+    def check_version(self, godot_version):
+        if self.dry_run:
+            print("Skipping version check in dry run mode (would likely fail)")
+            return
+        import importlib.util
+        version_file = os.path.join("git", "version.py")
+        spec = importlib.util.spec_from_file_location("version", version_file)
+        version = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(version)
+        if hasattr(version, "patch"):
+            version_string = f"{version.major}.{version.minor}.{version.patch}-{version.status}"
+        else:
+            version_string = f"{version.major}.{version.minor}-{version.status}"
+        ok = version_string == godot_version
+        if not ok:
+            print(f"Version mismatch, expected: {godot_version}, got: {version_string}")
+            sys.exit(1)
+
+    def checkout(self, ref):
+        repo = "https://github.com/godotengine/godot"
+        dest = os.path.join(self.base_dir, "git")
+        self.git("clone", dest, can_fail=True)
+        self.git("-C", dest, "fetch", "--all")
+        self.git("-C", dest, "checkout", "--detach", ref)
+
+    def tgz(self, version, ref="HEAD"):
+        source = os.path.join(self.base_dir, "git")
+        dest = os.path.join(self.base_dir, "godot.tar.gz")
+        self.git("-C", source, "archive", f"--prefix=godot-{version}/", "-o", dest, ref)

builder/cli.py

@@ -0,0 +1,172 @@
+import logging, os, sys
+from argparse import ArgumentParser
+
+from . import Config, ImageConfigs, GitRunner, PodmanRunner, write_config, load_config
+
+class ConfigCLI:
+    ACTION = "config"
+    HELP = "Print or save config file"
+
+    @staticmethod
+    def execute(base_dir, args):
+        write_config(sys.stdout)
+        sys.stdout.write('\n')
+        if args.save is not None:
+            path = args.save if os.path.isabs(args.save) else os.path.join(base_dir, args.save)
+            if not path.endswith(".json"):
+                print("Invalid config file: %s, must be '.json'" % args.save)
+                sys.exit(1)
+            with open(path, 'w') as w:
+                write_config(w)
+                print("Saved to file: %s" % path)
+
+    @staticmethod
+    def bind(parser):
+        parser.add_argument("-s", "--save")
+
+class ImageCLI:
+    ACTION = "fetch"
+    HELP = "Fetch remote build containers"
+
+    @staticmethod
+    def execute(base_dir, args):
+        podman = PodmanRunner(
+            base_dir,
+            dry_run=args.dry_run
+        )
+        podman.login()
+        podman.fetch_images(
+            images = args.image,
+            force=args.force_download
+        )
+
+    @staticmethod
+    def bind(parser):
+        parser.add_argument("-f", "--force-download", action="store_true")
+        parser.add_argument("-i", "--image", action="append", default=[], help="The image to fetch, all by default. Possible values: %s" % ", ".join(PodmanRunner.get_images()))
+
+
+class GitCLI:
+    ACTION = "checkout"
+    HELP = "git checkout, version check, tar"
+
+    @staticmethod
+    def execute(base_dir, args):
+        git = GitRunner(base_dir, dry_run=args.dry_run)
+        if not args.skip_checkout:
+            git.checkout(args.treeish)
+        if not args.skip_check:
+            git.check_version(args.godot_version)
+        if not args.skip_tar:
+            git.tgz(args.godot_version)
+
+    @staticmethod
+    def bind(parser):
+        parser.add_argument("treeish", help="git treeish, possibly a git ref, or commit hash.", default="origin/master")
+        parser.add_argument("godot_version", help="godot version (e.g. 3.1-alpha5)")
+        parser.add_argument("-c", "--skip-checkout", action="store_true")
+        parser.add_argument("-t", "--skip-tar", action="store_true")
+        parser.add_argument("--skip-check", action="store_true")
+
+
+class RunCLI:
+    ACTION = "run"
+    HELP = "Run the desired containers"
+
+    CONTAINERS = [cls.__name__.replace("Config", "") for cls in ImageConfigs]
+
+    @staticmethod
+    def execute(base_dir, args):
+        podman = PodmanRunner(base_dir, dry_run=args.dry_run)
+        build_mono = args.build == "all" or args.build == "mono"
+        build_classical = args.build == "all" or args.build == "classical"
+        if len(args.container) == 0:
+            args.container = RunCLI.CONTAINERS
+        to_build = [ImageConfigs[RunCLI.CONTAINERS.index(c)] for c in args.container]
+        for b in to_build:
+            podman.podrun(b, classical=build_classical, mono=build_mono, local=not args.remote, interactive=args.interactive)
+
+    def bind(parser):
+        parser.add_argument("-b", "--build", choices=["all", "classical", "mono"], default="all")
+        parser.add_argument("-k", "--container", action="append", default=[], help="The containers to build, one of %s" % RunCLI.CONTAINERS)
+        parser.add_argument("-r", "--remote", help="Run with remote containers", action="store_true")
+        parser.add_argument("-i", "--interactive", action="store_true", help="Enter an interactive shell inside the container instead of running the default command")
+
+
+class ReleaseCLI:
+    ACTION = "release"
+    HELP = "Make a full release cycle, git checkout, reset, version check, tar, build all"
+
+    @staticmethod
+    def execute(base_dir, args):
+        git = GitRunner(base_dir, dry_run=args.dry_run)
+        podman = PodmanRunner(base_dir, dry_run=args.dry_run)
+        build_mono = args.build == "all" or args.build == "mono"
+        build_classical = args.build == "all" or args.build == "classical"
+        if not args.localhost and not args.skip_download:
+            podman.login()
+            podman.fetch_images(
+                force=args.force_download
+            )
+        if not args.skip_git:
+            git.checkout(args.git)
+            git.check_version(args.godot_version)
+            git.tgz(args.godot_version)
+
+        for b in ImageConfigs:
+            podman.podrun(b, classical=build_classical, mono=build_mono, local=args.localhost)
+
+    @staticmethod
+    def bind(parser):
+        parser.add_argument("godot_version", help="godot version (e.g. 3.1-alpha5)")
+        parser.add_argument("-b", "--build", choices=["all", "classical", "mono"], default="all")
+        parser.add_argument("-s", "--skip-download", action="store_true")
+        parser.add_argument("-c", "--skip-git", action="store_true")
+        parser.add_argument("-g", "--git", help="git treeish, possibly a git ref, or commit hash.", default="origin/master")
+        parser.add_argument("-f", "--force-download", action="store_true")
+        parser.add_argument("-l", "--localhost", action="store_true")
+
+
+class CLI:
+    OPTS = [(v, getattr(Config, v)) for v in dir(Config) if not v.startswith("_")]
+
+    def add_command(self, cli):
+        parser = self.subparsers.add_parser(cli.ACTION, help=cli.HELP)
+        parser.add_argument("-n", "--dry-run", action="store_true")
+        parser.set_defaults(action_func=cli.execute)
+        cli.bind(parser)
+
+    def __init__(self, base_dir):
+        self.base_dir = base_dir
+        self.parser = ArgumentParser()
+        for k,v in CLI.OPTS:
+            self.parser.add_argument("--%s" % k)
+        self.parser.add_argument("-c", "--config", help="Configuration override")
+        self.subparsers = self.parser.add_subparsers(dest="action", help="The requested action", required=True)
+        self.add_command(ConfigCLI)
+        self.add_command(GitCLI)
+        self.add_command(ImageCLI)
+        self.add_command(RunCLI)
+        self.add_command(ReleaseCLI)
+
+    def execute(self):
+        args = self.parser.parse_args()
+        if args.config is not None:
+            path = args.config if os.path.isabs(args.config) else os.path.join(self.base_dir, args.config)
+            if not os.path.isfile(path):
+                print("Invalid config file: %s" % path)
+                sys.exit(1)
+            load_config(path)
+        for k,v in CLI.OPTS:
+            override = getattr(args, k)
+            if override is not None:
+                setattr(Config, k, override)
+        args.action_func(self.base_dir, args)
+
+
+def main(loglevel=logging.DEBUG):
+    logging.basicConfig(level=loglevel)
+    CLI(os.getcwd()).execute()
+
+if __name__ == "__main__":
+    main()