Skip to content

Commit f7bb0ca

Browse files
lyrixxlyrixx
authored
Merge pull request #450 from jolicode/build-services-separately
Build each service separately to reuse the registry cache
2 parents 5e6a08f + adeb248 commit f7bb0ca

2 files changed

Lines changed: 112 additions & 31 deletions

File tree

‎.castor/docker.php‎

Lines changed: 111 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
use Castor\Helper\PathHelper;
1010
use Symfony\Component\Console\Completion\CompletionInput;
1111
use Symfony\Component\Console\Input\InputOption;
12+
use Symfony\Component\Console\Output\OutputInterface;
1213
use Symfony\Component\Process\Exception\ExceptionInterface;
1314
use Symfony\Component\Process\Exception\ProcessFailedException;
1415
use Symfony\Component\Process\ExecutableFinder;
@@ -22,6 +23,7 @@
2223
use function Castor\http_client;
2324
use function Castor\io;
2425
use function Castor\open;
26+
use function Castor\parallel;
2527
use function Castor\run;
2628
use function Castor\variable;
2729
use function worktree\get_worktree_name;
@@ -105,27 +107,90 @@ function build(
105107

106108
io()->title('Building infrastructure');
107109

108-
$command = [];
109-
110-
$command[] = '--profile';
111-
if ($profile) {
112-
$command[] = $profile;
113-
} else {
114-
$command[] = '*';
115-
}
116-
117110
$command = [
118-
...$command,
111+
'--profile', $profile ?: '*',
119112
'build',
120113
'--build-arg', 'PHP_VERSION=' . variable('php_version'),
121114
'--build-arg', 'PROJECT_NAME=' . variable('project_name'),
122115
];
123116

124117
if ($service) {
125-
$command[] = $service;
118+
docker_compose([...$command, $service]);
119+
120+
return;
126121
}
127122

128-
docker_compose($command);
123+
// One build per service, in parallel, rather than all of them at once: in a single
124+
// BuildKit session, services sharing a stage (e.g. php-base) lose the cache of all
125+
// but one of them, see https://github.com/moby/buildkit/issues/6418
126+
$services = array_filter(get_services($profile), static fn (array $config) => isset($config['build']));
127+
128+
// The services with a cache first, the others once they are built: a service
129+
// without cache sharing their stages (e.g. a worker) would build these stages from
130+
// scratch meanwhile, and these local layers prevent the others from finding the
131+
// next steps in their imported cache
132+
$waves = [
133+
array_keys(array_filter($services, static fn (array $config) => isset($config['build']['cache_from']))),
134+
array_keys(array_filter($services, static fn (array $config) => !isset($config['build']['cache_from']))),
135+
];
136+
137+
$c = context()->withPty(false)->withTty(false);
138+
139+
foreach ($waves as $wave) {
140+
run_builds_in_parallel(array_combine($wave, array_map(
141+
static fn (string $service) => static fn (callable $callback) => docker_compose([...$command, $service], $c, callback: $callback),
142+
$wave,
143+
)));
144+
}
145+
}
146+
147+
/**
148+
* Runs the builds in parallel, prefixing each line of their output with the name of
149+
* their service: their outputs are mixed otherwise.
150+
*
151+
* @param array<string, \Closure(callable(string, string, Process): void): Process> $builds
152+
*/
153+
function run_builds_in_parallel(array $builds): void
154+
{
155+
if (!$builds) {
156+
return;
157+
}
158+
159+
$width = max(array_map(strlen(...), array_keys($builds)));
160+
161+
parallel(...array_map(
162+
static fn (string $service, \Closure $build) => static function () use ($service, $build, $width): void {
163+
$prefix = str_pad("[{$service}]", $width + 2) . ' ';
164+
$write = static function (string $lines) use ($prefix): void {
165+
foreach (explode("\n", $lines) as $line) {
166+
// BuildKit separates its steps with empty lines, useless once prefixed
167+
if ('' !== trim($line)) {
168+
io()->write($prefix . $line . "\n", false, OutputInterface::OUTPUT_RAW);
169+
}
170+
}
171+
};
172+
173+
// Only complete lines are written, the end of each stream is kept until its next chunk
174+
$buffers = [Process::OUT => '', Process::ERR => ''];
175+
176+
try {
177+
$build(static function (string $type, string $bytes) use (&$buffers, $write): void {
178+
$buffers[$type] .= str_replace("\r\n", "\n", $bytes);
179+
180+
if (false !== $position = strrpos($buffers[$type], "\n")) {
181+
$write(substr($buffers[$type], 0, $position));
182+
$buffers[$type] = substr($buffers[$type], $position + 1);
183+
}
184+
});
185+
} finally {
186+
foreach (array_filter($buffers) as $buffer) {
187+
$write($buffer);
188+
}
189+
}
190+
},
191+
array_keys($builds),
192+
$builds,
193+
));
129194
}
130195

131196
/**
@@ -408,10 +473,11 @@ function get_compose_environment(Context $c): array
408473
}
409474

410475
/**
411-
* @param list<string> $subCommand
412-
* @param list<string> $profiles
476+
* @param list<string> $subCommand
477+
* @param list<string> $profiles
478+
* @param (callable(string, string, Process): void)|null $callback Receives the output instead of the console
413479
*/
414-
function docker_compose(array $subCommand, ?Context $c = null, array $profiles = []): Process
480+
function docker_compose(array $subCommand, ?Context $c = null, array $profiles = [], ?callable $callback = null): Process
415481
{
416482
$c ??= context();
417483
$profiles = $profiles ?: ['default'];
@@ -451,7 +517,7 @@ function docker_compose(array $subCommand, ?Context $c = null, array $profiles =
451517

452518
$command = array_merge($command, $subCommand);
453519

454-
return run($command, context: $c);
520+
return run($command, context: $c, callback: $callback);
455521
}
456522

457523
/**
@@ -614,35 +680,49 @@ function push(
614680
$command[] = '--set';
615681
$command[] = '*.args.PHP_VERSION=' . $c['php_version'];
616682

683+
if ($dryRun) {
684+
$command[] = '--print';
685+
}
686+
687+
// One bake per service, see build()
688+
$commands = [];
617689
foreach ($cacheFroms as $service => $cacheFrom) {
618-
$command[] = '--set';
619-
$command[] = "{$service}.cache-to={$cacheFrom},mode=max";
690+
$serviceCommand = [...$command, '--set', "{$service}.cache-to={$cacheFrom},mode=max"];
620691

621-
if (!$tag) {
622-
continue;
623-
}
692+
if ($tag) {
693+
// Image name without its tag, e.g. "ghcr.io/jolicode/docker-starter/php"
694+
$image = isset($services[$service]['image']) ? preg_replace('{:[^/]+$}', '', $services[$service]['image']) : "{$registry}/{$service}";
624695

625-
// Image name without its tag, e.g. "ghcr.io/jolicode/docker-starter/php"
626-
$image = isset($services[$service]['image']) ? preg_replace('{:[^/]+$}', '', $services[$service]['image']) : "{$registry}/{$service}";
696+
foreach ($tag as $t) {
697+
$serviceCommand[] = '--set';
698+
$serviceCommand[] = "{$service}.tags={$image}:{$t}";
699+
}
627700

628-
foreach ($tag as $t) {
629-
$command[] = '--set';
630-
$command[] = "{$service}.tags={$image}:{$t}";
701+
$serviceCommand[] = '--set';
702+
$serviceCommand[] = "{$service}.output=type=registry";
631703
}
632704

633-
$command[] = '--set';
634-
$command[] = "{$service}.output=type=registry";
705+
$commands[$service] = [...$serviceCommand, $service];
635706
}
636707

637708
if ($dryRun) {
638-
$command[] = '--print';
709+
foreach ($commands as $serviceCommand) {
710+
run($serviceCommand, context: $c);
711+
}
712+
713+
return;
639714
}
640715

641-
run([...$command, ...array_keys($cacheFroms)], context: $c);
716+
$c = $c->withPty(false)->withTty(false);
717+
718+
run_builds_in_parallel(array_map(
719+
static fn (array $serviceCommand) => static fn (callable $callback) => run($serviceCommand, context: $c, callback: $callback),
720+
$commands,
721+
));
642722
}
643723

644724
/**
645-
* @return array<string, array{image?: string, profiles?: list<string>, build: array{context: string, dockerfile?: string, cache_from?: list<string>, target?: string, additional_contexts?: array<string, string>}}>
725+
* @return array<string, array{image?: string, profiles?: list<string>, build?: array{context: string, dockerfile?: string, cache_from?: list<string>, target?: string, additional_contexts?: array<string, string>}}>
646726
*/
647727
function get_services(?string $profile = null): array
648728
{

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
* Mount the project in `/var/www` instead of `/home/app`
3030
* Add git worktree support (auto-isolated project name, ports, volumes, networks)
3131
* Add support for caching image cache in a registry
32+
* Build (and push the cache of) each service separately, in parallel, so services sharing a stage all reuse their cache
3233
* Add production images (`php` and `nginx`): code baked in, non-root, php-fpm on a unix socket
3334
* Share php-fpm and nginx configuration between the dev `frontend` container and the production images
3435
* The `frontend` container now listens on port 8080 and php-fpm on a unix socket

0 commit comments

Comments
 (0)