From e396b477daaa3bfd7ba2ceebfdaba7c3d0ebd255 Mon Sep 17 00:00:00 2001 From: Marten Richter Date: Sun, 27 Jul 2025 10:03:54 +0000 Subject: [PATCH] Add basic support for squashfs --- empack/cli/pack.py | 13 +++++- empack/pack.py | 94 +++++++++++++++++++++++++++++++++--------- empack/sqshfs_utils.py | 50 ++++++++++++++++++++++ 3 files changed, 137 insertions(+), 20 deletions(-) create mode 100644 empack/sqshfs_utils.py diff --git a/empack/cli/pack.py b/empack/cli/pack.py index e1194fc..117bf2a 100644 --- a/empack/cli/pack.py +++ b/empack/cli/pack.py @@ -45,6 +45,16 @@ def pack_env_cli( "-c", help="path to a .yaml file with the empack config", ), + package_squashfs: Optional[bool] = typer.Option( # noqa: B008 + False, + "--package-squashfs/--no-package-squashfs", + help="package each package into squashfs", + ), + environment_squashfs: Optional[bool] = typer.Option( # noqa: B008 + False, + "--environment-squashfs/--no-environment-squashfs", + help="package the whole environment into one squashfs", + ), use_cache: Optional[bool] = typer.Option( # noqa: B008 True, "--use-cache/--no-use-cache", @@ -66,11 +76,12 @@ def pack_env_cli( ), ): file_filters = pkg_file_filter_from_yaml(*config) - pack_env( env_prefix=env_prefix, relocate_prefix=relocate_prefix, file_filters=file_filters, + package_squashfs=package_squashfs, + environment_squashfs=environment_squashf, outdir=outdir, cache_dir=cache_dir, use_cache=use_cache, diff --git a/empack/pack.py b/empack/pack.py index e4440b9..c36b9ee 100644 --- a/empack/pack.py +++ b/empack/pack.py @@ -14,6 +14,7 @@ from .filter_env import filter_env, filter_pkg, iterate_env_pkg_meta from .micromamba_wrapper import create_environment from .tar_utils import ALLOWED_FORMATS, save_as_tarfile +from .sqshfs_utils import save_as_squashfs EMPACK_CACHE_DIR = Path(user_cache_dir("empack")) PACKED_PACKAGES_CACHE_DIR = EMPACK_CACHE_DIR / "packed_packages_cache" @@ -117,13 +118,22 @@ def pack_pkg_impl( pkg_meta, compression_format, use_cache, + package_squashfs, + environment_squashfs, compresslevel, cache_dir=None, outdir=None, ): if cache_dir is None: cache_dir = PACKED_PACKAGES_CACHE_DIR - fname_core = f"{filename_base_from_meta(pkg_meta)}.tar.{compression_format}" + if (package_squashfs or environment_squashfs) and relocate_prefix!="/": + raise ValueError("Other relocate prefixes than '/' are not supported") + if package_squashfs: + fname_core = f"{filename_base_from_meta(pkg_meta)}.sqshfs" + elif environment_squashfs: + fname_core = f"environment.sqshfs" + else: + fname_core = f"{filename_base_from_meta(pkg_meta)}.tar.{compression_format}" cache_file = cache_dir / fname_core fname = os.path.join(outdir, fname_core) if outdir is not None else fname_core @@ -132,28 +142,38 @@ def pack_pkg_impl( if outdir is not None: shutil.copy(cache_file, fname) return fname_core, True - conda_meta_filename = f"{filename_base_from_meta(pkg_meta)}.json" with open(filtered_prefix / "conda-meta" / conda_meta_filename, "w") as f: json.dump(pkg_meta, f) - # make included files absolute - filenames = [os.path.join(filtered_prefix, f) for f in included_files] - arcnames = [os.path.join(relocate_prefix, f) for f in included_files] - # arcnames relative to relocate_prefix - arcnames = [os.path.relpath(a, relocate_prefix) for a in arcnames] + if not environment_squashfs: + if not package_squashfs: + # make included files absolute + filenames = [os.path.join(filtered_prefix, f) for f in included_files] + arcnames = [os.path.join(relocate_prefix, f) for f in included_files] - # compress the filtered environment - save_as_tarfile( - output_filename=fname, - filenames=filenames, - arcnames=arcnames, - compression_format=compression_format, - compresslevel=compresslevel, - ) - # copy to cache - shutil.copy(fname, cache_file) + # arcnames relative to relocate_prefix + arcnames = [os.path.relpath(a, relocate_prefix) for a in arcnames] + + # compress the filtered environment + save_as_tarfile( + output_filename=fname, + filenames=filenames, + arcnames=arcnames, + compression_format=compression_format, + compresslevel=compresslevel, + ) + else: + filenames = list(included_files) + save_as_squashfs( + output_filename=fname, + filtered_prefix=filtered_prefix, + filenames=filenames, + compresslevel=compresslevel + ) + # copy to cache + shutil.copy(fname, cache_file) return fname_core, False @@ -256,6 +276,8 @@ def pack_env( file_filters, use_cache, cache_dir=None, + package_squashfs=False, + environment_squashfs=True, compression_format=ALLOWED_FORMATS[0], compresslevel=9, outdir=None, @@ -269,8 +291,14 @@ def pack_env( target_dir=filtered_prefix, pkg_file_filter=file_filters, ) + if package_squashfs and environment_squashfs: + raise ValueError( + "You can not pack the whole environment in one squash file" + " and compress the packages to individual squashfs files at the same time.") + packages_info = [] + environment_files = [] for pkg_meta in iterate_env_pkg_meta(filtered_prefix): pack_pkg_impl( included_files=included_files[pkg_meta["name"]], @@ -280,9 +308,14 @@ def pack_env( use_cache=use_cache, outdir=outdir, cache_dir=cache_dir, + package_squashfs=package_squashfs, + environment_squashfs=environment_squashfs, compression_format=compression_format, - compresslevel=compresslevel, + compresslevel=compresslevel ) + if environment_squashfs: + environment_files.extend(included_files[pkg_meta["name"]]) + base_fname = filename_base_from_meta(pkg_meta) @@ -291,7 +324,15 @@ def pack_env( version=pkg_meta["version"], build=pkg_meta["build"], filename_stem=base_fname, - filename=f"{base_fname}.tar.{compression_format}", + filename=( + f"{base_fname}.tar.{compression_format}" + if not package_squashfs and not environment_squashfs + else ( + f"{base_fname}.sqshfs" + if not environment_squashfs + else f"environment.sqshfs" + ) + ), channel=pkg_meta["channel"], depends=pkg_meta["depends"], subdir=pkg_meta["subdir"], @@ -303,6 +344,21 @@ def pack_env( if package_url is not None: pkg_dict["url"] = package_url packages_info.append(pkg_dict) + + if environment_squashfs: + env_file_core = f"environment.sqshfs" + env_file = os.path.join(outdir, env_file_core) if outdir is not None else env_file_core + save_as_squashfs( + output_filename=env_file, + filtered_prefix=filtered_prefix, + filenames=environment_files, + compresslevel=compresslevel + ) + if cache_dir is None: + cache_dir = PACKED_PACKAGES_CACHE_DIR + cache_file = cache_dir / env_file_core + # copy to cache + shutil.copy(env_file, cache_file) # save the list of packages env_meta = { diff --git a/empack/sqshfs_utils.py b/empack/sqshfs_utils.py new file mode 100644 index 0000000..b7cf477 --- /dev/null +++ b/empack/sqshfs_utils.py @@ -0,0 +1,50 @@ +import os +import subprocess +from pathlib import Path + + +def save_as_squashfs( + output_filename, + filtered_prefix, + filenames, + compresslevel=9 +): + if not Path(output_filename).parts[-1].endswith(f".sqshfs"): + error_message = ( + f"Output filename {output_filename} does not end with .sqshfs" + ) + raise RuntimeError(error_message) + + try: + os.chdir(filtered_prefix) + mksquashfs_command = ['mksquashfs', '-', output_filename, '-cpiostyle0', '-b', '128K', '-comp', 'zstd', '-noappend'] + # print("peak mksq", mksquashfs_command) + process = subprocess.Popen( + mksquashfs_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=filtered_prefix + ) + # Note: we may want to string c++ source files? + for filename in filenames: + # print("peakfilename",filtered_prefix, filename) + process.stdin.write(str(filename).encode() + b'\0') + + stdout, stderr = process.communicate() + process.stdin.close() + + if stdout: + print(f"mksquashfs stdout:\n{stdout.decode()}") + if stderr: + print(f"mksquashfs stderr:\n{stderr.decode()}") + if process.returncode != 0: + raise subprocess.CalledProcessError( + process.returncode, + mksquashfs_command, + output=stdout, + stderr=stderr + ) + + except FileNotFoundError: + print("Error: 'mksquashfs' command not found. Install it with conda install squashfs-tools")