Feat/type 3 format

Pipfile

@@ -16,6 +16,7 @@ emrichen = "*"
 shutils = "*"
 "ruamel.yaml" = "*"
 "roam" = "*"
+bson = "*"
 
 [requires]
 python_version = "3.8"

appimagebuilder/__main__.py

@@ -12,18 +12,6 @@
 #  The above copyright notice and this permission notice shall be included in
 #  all copies or substantial portions of the Software.
 
-#  Copyright  2020 Alexis Lopez Zubieta
-#
-#  Permission is hereby granted, free of charge, to any person obtaining a
-#  copy of this software and associated documentation files (the "Software"),
-#  to deal in the Software without restriction, including without limitation the
-#  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-#  sell copies of the Software, and to permit persons to whom the Software is
-#  furnished to do so, subject to the following conditions:
-#
-#  The above copyright notice and this permission notice shall be included in
-#  all copies or substantial portions of the Software.
-
 import logging
 
 from appimagebuilder import recipe

appimagebuilder/commands/create_appimage.py

@@ -9,8 +9,11 @@
 #
 #  The above copyright notice and this permission notice shall be included in
 #  all copies or substantial portions of the Software.
-from appimagebuilder.modules.appimage import AppImageCreator
+import os
+
+from appimagebuilder.modules.prime.type_2 import Type2Creator
 from appimagebuilder.commands.command import Command
+from appimagebuilder.modules.prime.type_3 import Type3Creator
 from appimagebuilder.recipe.roamer import Roamer
 
 
@@ -23,5 +26,52 @@ def id(self):
         super().id()
 
     def __call__(self, *args, **kwargs):
-        creator = AppImageCreator(self.recipe)
+        appimage_format = self.recipe.AppImage.format() or 2
+        self.app_dir = self.recipe.AppDir.path()
+
+        self.target_arch = self.recipe.AppImage.arch()
+        self.app_name = self.recipe.AppDir.app_info.name()
+        self.app_version = self.recipe.AppDir.app_info.version()
+
+        fallback_file_name = os.path.join(
+            os.getcwd(),
+            "%s-%s-%s.AppImage" % (self.app_name, self.app_version, self.target_arch),
+        )
+        self.file_name = self.recipe.AppDir.app_info.file_name() or fallback_file_name
+
+        if appimage_format == 2:
+            self._create_type_2_appimage()
+            return
+
+        if appimage_format == 3:
+            self._create_type_3_appimage()
+            return
+
+        raise RuntimeError(f"Unknown AppImage format {appimage_format}")
+
+    def _create_type_2_appimage(self):
+        update_information = self.recipe.AppImage["update-information"]() or "None"
+
+        sign_key = self.recipe.AppImage["sign-key"] or "None"
+        if sign_key == "None":
+            sign_key = None
+        creator = Type2Creator(
+            self.app_dir,
+            self.target_arch,
+            update_information,
+            sign_key,
+            self.file_name,
+        )
         creator.create()
+
+    def _create_type_3_appimage(self):
+        update_information = self.recipe.AppImage["update-information"]() or "None"
+        creator = Type3Creator(self.app_dir)
+        creator.create(
+            self.file_name,
+            {
+                "update-information": update_information
+            },
+            gnupg_key=self.recipe.AppImage["sign-key"]() or None,
+            compression_method="zstd",
+        )

appimagebuilder/modules/prime/__init__.py

@@ -0,0 +1,11 @@
+#  Copyright  2021 Alexis Lopez Zubieta
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of this software and associated documentation files (the "Software"),
+#  to deal in the Software without restriction, including without limitation the
+#  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+#  sell copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.

appimagebuilder/modules/prime/common.py

@@ -0,0 +1,20 @@
+#  Copyright  2021 Alexis Lopez Zubieta
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of this software and associated documentation files (the "Software"),
+#  to deal in the Software without restriction, including without limitation the
+#  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+#  sell copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+import logging
+import os
+from urllib import request
+
+
+def download_if_required(url, path):
+    if not os.path.exists(path):
+        logging.info("Downloading: %s" % url)
+        request.urlretrieve(url, path)

appimagebuilder/modules/appimage.py → appimagebuilder/modules/prime/type_2.py

@@ -1,3 +1,15 @@
+#  Copyright  2021 Alexis Lopez Zubieta
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of this software and associated documentation files (the "Software"),
+#  to deal in the Software without restriction, including without limitation the
+#  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+#  sell copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+
 #  Copyright  2020 Alexis Lopez Zubieta
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a
@@ -14,15 +26,14 @@
 from urllib import request
 
 from appimagebuilder.gateways.appimagetool import AppImageToolCommand
+from appimagebuilder.modules.prime import common
 
 
-class AppImageCreator:
-    def __init__(self, recipe):
-        self.app_dir = recipe.AppDir.path()
-        self.target_arch = recipe.AppImage.arch()
-        self.app_name = recipe.AppDir.app_info.name()
-        self.app_version = recipe.AppDir.app_info.version()
-        self.update_information = recipe.AppImage["update-information"]() or "None"
+class Type2Creator:
+    def __init__(self, appdir, target_arch, update_information, sign_key, output_filename):
+        self.app_dir = appdir
+        self.target_arch = target_arch
+        self.update_information = update_information
         self.guess_update_information = False
 
         if self.update_information == "None":
@@ -34,22 +45,18 @@ def __init__(self, recipe):
             self.update_information = None
             self.guess_update_information = True
 
-        self.sing_key = recipe.AppImage["sign-key"]() or "None"
+        self.sing_key = sign_key
         if self.sing_key == "None":
             self.sing_key = None
 
-        fallback_file_name = os.path.join(
-            os.getcwd(),
-            "%s-%s-%s.AppImage" % (self.app_name, self.app_version, self.target_arch),
-        )
-        self.target_file = recipe.AppImage.file_name() or fallback_file_name
+        self.target_file = output_filename
 
     def create(self):
         self._assert_target_architecture()
 
         runtime_url = self._get_runtime_url()
         runtime_path = self._get_runtime_path()
-        self._download_runtime_if_required(runtime_path, runtime_url)
+        common.download_if_required(runtime_url, runtime_path)
 
         self._generate_appimage(runtime_path)
 
@@ -62,11 +69,6 @@ def _generate_appimage(self, runtime_path):
         appimage_tool.runtime_file = runtime_path
         appimage_tool.run()
 
-    def _download_runtime_if_required(self, runtime_path, runtime_url):
-        if not os.path.exists(runtime_path):
-            logging.info("Downloading runtime: %s" % runtime_url)
-            request.urlretrieve(runtime_url, runtime_path)
-
     def _get_runtime_path(self):
         os.makedirs("appimage-builder-cache", exist_ok=True)
         runtime_path = "appimage-builder-cache/runtime-%s" % self.target_arch

appimagebuilder/modules/prime/type_3.py

@@ -0,0 +1,187 @@
+#  Copyright  2021 Alexis Lopez Zubieta
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of this software and associated documentation files (the "Software"),
+#  to deal in the Software without restriction, including without limitation the
+#  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+#  sell copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+import logging
+import os
+import pathlib
+import subprocess
+import tempfile
+
+import bson
+
+from appimagebuilder.modules.prime import common
+from appimagebuilder.utils import file_utils
+from appimagebuilder.utils import shell, elf
+
+
+class Type3Creator:
+    def __init__(self, app_dir, cache_dir="appimage-builder-cache"):
+        self.logger = logging.getLogger()
+        self.app_dir = pathlib.Path(app_dir).absolute()
+        self.cache_dir = pathlib.Path(cache_dir)
+        self.runtime_project_url = (
+            "https://github.com/AppImageCrafters/appimage-runtime"
+        )
+
+        self.required_tool_paths = shell.resolve_commands_paths(["mksquashfs", "gpg"])
+
+    def create(
+            self, output_filename, metadata=None, gnupg_key=None, compression_method="gzip"
+    ):
+        if metadata is None:
+            metadata = {}
+
+        self.logger.warning(
+            "Type 3 AppImages are still experimental and under development!"
+        )
+
+        squashfs_path = self._squash_appdir(compression_method)
+
+        runtime_path = self._resolve_executable()
+
+        file_utils.extend_file(runtime_path, squashfs_path, output_filename)
+
+        payload_offset = os.path.getsize(runtime_path)
+        metadata_offset = os.path.getsize(output_filename)
+        self._append_metadata(output_filename, metadata)
+        signatures_offset = os.path.getsize(output_filename)
+        self._fill_header(
+            output_filename, payload_offset, metadata_offset, signatures_offset
+        )
+
+        if gnupg_key:
+            self._sign_bundle(output_filename, gnupg_key, signatures_offset)
+
+        # remove squashfs
+        squashfs_path.unlink()
+
+        file_utils.set_permissions_rx_all(output_filename)
+
+    def _squash_appdir(self, compression_method):
+        squashfs_path = self.cache_dir / "AppDir.sqfs"
+
+        self.logger.info("Squashing AppDir")
+        command = "{mksquashfs} {AppDir} {squashfs_path} -reproducible -comp {compression} ".format(
+            AppDir=self.app_dir,
+            squashfs_path=squashfs_path,
+            compression=compression_method,
+            **self.required_tool_paths,
+        )
+        _proc = subprocess.run(
+            command,
+            stderr=subprocess.PIPE,
+            shell=True,
+        )
+
+        shell.assert_successful_result(_proc)
+        return squashfs_path
+
+    def _resolve_executable(self):
+        launcher_arch = elf.get_arch(self.app_dir / "AppRun")
+        url = self._get_runtime_url(launcher_arch)
+        path = self._get_runtime_path(launcher_arch)
+        common.download_if_required(url, path.__str__())
+
+        return path
+
+    def _fill_header(
+            self, output_filename, payload_offset, metadata_offset, signature_offset
+    ):
+        with open(output_filename, "r+b") as f:
+            f.seek(0x410, 0)
+            f.write(payload_offset.to_bytes(8, "little"))
+            f.write(metadata_offset.to_bytes(8, "little"))
+            f.write(signature_offset.to_bytes(8, "little"))
+
+    def _get_runtime_path(self, arch):
+        self.cache_dir.parent.mkdir(parents=True, exist_ok=True)
+        runtime_path = self.cache_dir / f"runtime-{arch}"
+
+        return runtime_path
+
+    def _get_runtime_url(self, arch):
+        runtime_url_template = (
+                self.runtime_project_url
+                + "/releases/download/continuous/runtime-Release-%s"
+        )
+        runtime_url = runtime_url_template % arch
+        return runtime_url
+
+    def _append_metadata(self, output_filename, metadata):
+        raw = bson.dumps(metadata)
+
+        with open(output_filename, "r+b") as fd:
+            fd.seek(0, 2)
+            fd.write(raw)
+
+    def _sign_bundle(self, output_filename, keyid, signatures_offset):
+        signature = self._generate_bundle_signature_using_gpg(
+            keyid, output_filename, signatures_offset
+        )
+
+        encoded_signatures = bson.dumps(
+            {
+                "signatures": [
+                    {
+                        "method": "gpg",
+                        "keyid": keyid,
+                        "data": signature,
+                    }
+                ]
+            }
+        )
+
+        with open(output_filename, "r+b") as fd:
+            fd.seek(signatures_offset, 0)
+            fd.write(encoded_signatures)
+
+    def _generate_bundle_signature_using_gpg(self, keyid, filename, limit):
+        # file chunks will be written here
+        input_path = tempfile.NamedTemporaryFile().name
+        os.mkfifo(input_path)
+
+        # sign the file with out including the signatures section
+        output_path = tempfile.NamedTemporaryFile().name
+
+        # call gpg
+        args = [
+            self.required_tool_paths["gpg"],
+            "--detach-sign",
+            "--armor",
+            "--default-key",
+            keyid,
+            "--output",
+            output_path,
+            input_path,
+        ]
+
+        with subprocess.Popen(args) as _proc:
+            # read file contents up to limit
+            with open(input_path, "wb") as input_pipe:
+                chunk_size = 1024
+                n_chunks = int(limit / chunk_size)
+                with open(filename, "rb") as input_file:
+                    for chunk_id in range(n_chunks):
+                        input_pipe.write(input_file.read(chunk_size))
+
+                    final_chunk_size = limit - (n_chunks * chunk_size)
+                    if final_chunk_size != 0:
+                        input_pipe.write(input_file.read(final_chunk_size))
+
+                    input_pipe.close()
+
+        # read output
+        with open(output_path, "rb") as output:
+            signature = output.read().decode()
+
+        os.unlink(output_path)
+        os.unlink(input_path)
+        return signature