From 27ca4a65756be018c84bea22da4cf5c1f18da5ef Mon Sep 17 00:00:00 2001 From: Gareth Latty Date: Sun, 28 Apr 2019 03:44:04 +0100 Subject: [PATCH] Add ZiX-12A support. --- README.md | 2 +- setup.py | 4 +- unrpa/__init__.py | 9 +- unrpa/__main__.py | 272 ++++++++++++++++++++++-------------------- unrpa/versions/zix.py | 173 +++++++++++++++------------ unrpa/view.py | 9 +- 6 files changed, 251 insertions(+), 218 deletions(-) diff --git a/README.md b/README.md index 094d604..0f60357 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ usage: unrpa [-h] [-v] [-s] [-l] [-p PATH] [-m] [-f VERSION] | -l, --list | only list contents, do not extract. | | -p PATH, --path PATH | will extract to the given path. | | -m, --mkdir | will make any non-existent directories in extraction path. | -| -f VERSION, --force VERSION | forces an archive version. May result in failure.
Possible versions: RPA-3.0, ZiX-12B, ALT-1.0, RPA-2.0, RPA-1.0. | +| -f VERSION, --force VERSION | forces an archive version. May result in failure.
Possible versions: RPA-3.0, ZiX-12A, ZiX-12B, ALT-1.0, RPA-2.0, RPA-1.0. | | --continue-on-error | try to continue extraction when something goes wrong. | | -o OFFSET, --offset OFFSET | sets an offset to be used to decode unsupported archives. | | -k KEY, --key KEY | sets a key to be used to decode unsupported archives. | diff --git a/setup.py b/setup.py index 06a89bf..4cad8b4 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="unrpa", - version="2.0.0", + version="2.0.1", author="Gareth Latty", author_email="gareth@lattyware.co.uk", description="Extract files from the RPA archive format (from the Ren'Py Visual Novel Engine).", @@ -22,5 +22,5 @@ "Operating System :: OS Independent", "Environment :: Console", ], - entry_points={"console_scripts": ["unrpa = unrpa:__main__"]}, + entry_points={"console_scripts": ["unrpa = unrpa.__main__:main"]}, ) diff --git a/unrpa/__init__.py b/unrpa/__init__.py index 591cf86..e06e5c4 100755 --- a/unrpa/__init__.py +++ b/unrpa/__init__.py @@ -1,4 +1,3 @@ -import io import os import pickle import sys @@ -100,11 +99,7 @@ def extract_files(self) -> None: os.path.join(self.path, os.path.split(path)[0]) ) file_view = self.extract_file( - path, - data, - file_number, - total_files, - cast(io.BufferedReader, archive), + path, data, file_number, total_files, archive ) with open(os.path.join(self.path, path), "wb") as output_file: version.postprocess(file_view, output_file) @@ -131,7 +126,7 @@ def extract_file( data: ComplexIndexEntry, file_number: int, total_files: int, - archive: io.BufferedIOBase, + archive: BinaryIO, ) -> ArchiveView: self.log( UnRPA.info, f"[{file_number / float(total_files):04.2%}] {name:>3}", name diff --git a/unrpa/__main__.py b/unrpa/__main__.py index 9252842..9446aec 100644 --- a/unrpa/__main__.py +++ b/unrpa/__main__.py @@ -25,140 +25,152 @@ from unrpa import UnRPA from unrpa.errors import UnRPAError -parser = argparse.ArgumentParser( - prog="unrpa", - description="Extract files from the RPA archive format (from the Ren'Py Visual Novel Engine).", -) - -parser.add_argument( - "-v", - "--verbose", - action="count", - dest="verbose", - default=1, - help="explain what is being done [default].", -) -parser.add_argument( - "-s", "--silent", action="store_const", const=0, dest="verbose", help="no output." -) -parser.add_argument( - "-l", - "--list", - action="store_true", - dest="list", - default=False, - help="only list contents, do not extract.", -) -parser.add_argument( - "-p", - "--path", - action="store", - type=str, - dest="path", - default=None, - help="will extract to the given path.", -) -parser.add_argument( - "-m", - "--mkdir", - action="store_true", - dest="mkdir", - default=False, - help="will make any non-existent directories in extraction path.", -) -parser.add_argument( - "-f", - "--force", - action="store", - type=str, - dest="version", - default=None, - help="forces an archive version. May result in failure. Possible versions: " - + ", ".join(version.name for version in UnRPA.provided_versions) - + ".", -) -parser.add_argument( - "--continue-on-error", - action="store_true", - dest="continue_on_error", - default=False, - help="try to continue extraction when something goes wrong.", -) -parser.add_argument( - "-o", - "--offset", - action="store", - type=int, - dest="offset", - default=None, - help="sets an offset to be used to decode unsupported archives.", -) -parser.add_argument( - "-k", - "--key", - action="store", - type=int, - dest="key", - default=None, - help="sets a key to be used to decode unsupported archives.", -) - -parser.add_argument("--version", action="version", version="%(prog)s 2.0.0") - -parser.add_argument( - "filename", metavar="FILENAME", type=str, help="the RPA file to extract." -) - -args: Any = parser.parse_args() - -provided_version = None -if args.version: - try: - provided_version = next( - version - for version in UnRPA.provided_versions - if args.version.lower() == version.name.lower() - ) - except StopIteration: - parser.error( - "The archive version you gave isn’t one we recognise - it needs to be one of: " - + ", ".join(version.name for version in UnRPA.provided_versions) - ) -provided_offset_and_key: Optional[Tuple[int, int]] = None -if args.key and args.offset: - provided_offset_and_key = (args.offset, args.key) -elif bool(args.key) != bool(args.offset): - parser.error("If you set --key or --offset, you must set both.") +def main() -> None: + parser = argparse.ArgumentParser( + prog="unrpa", + description="Extract files from the RPA archive format (from the Ren'Py Visual Novel Engine).", + ) -if args.list and args.path: - parser.error("Option -path: only valid when extracting.") + parser.add_argument( + "-v", + "--verbose", + action="count", + dest="verbose", + default=1, + help="explain what is being done [default].", + ) + parser.add_argument( + "-s", + "--silent", + action="store_const", + const=0, + dest="verbose", + help="no output.", + ) + parser.add_argument( + "-l", + "--list", + action="store_true", + dest="list", + default=False, + help="only list contents, do not extract.", + ) + parser.add_argument( + "-p", + "--path", + action="store", + type=str, + dest="path", + default=None, + help="will extract to the given path.", + ) + parser.add_argument( + "-m", + "--mkdir", + action="store_true", + dest="mkdir", + default=False, + help="will make any non-existent directories in extraction path.", + ) + parser.add_argument( + "-f", + "--force", + action="store", + type=str, + dest="version", + default=None, + help="forces an archive version. May result in failure. Possible versions: " + + ", ".join(version.name for version in UnRPA.provided_versions) + + ".", + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + dest="continue_on_error", + default=False, + help="try to continue extraction when something goes wrong.", + ) + parser.add_argument( + "-o", + "--offset", + action="store", + type=int, + dest="offset", + default=None, + help="sets an offset to be used to decode unsupported archives.", + ) + parser.add_argument( + "-k", + "--key", + action="store", + type=int, + dest="key", + default=None, + help="sets a key to be used to decode unsupported archives.", + ) -if args.mkdir and not args.path: - parser.error("Option --mkdir: only valid when --path is set.") + parser.add_argument("--version", action="version", version="%(prog)s 2.0.1") -if not args.mkdir and args.path and not os.path.isdir(args.path): - parser.error(f"No such directory: “{args.path}”. Use --mkdir to create it.") + parser.add_argument( + "filename", metavar="FILENAME", type=str, help="the RPA file to extract." + ) -if args.list and args.verbose == 0: - parser.error("Option --list: can’t be silent while listing data.") + args: Any = parser.parse_args() -if not os.path.isfile(args.filename): - parser.error(f"No such file: “{args.filename}”.") + provided_version = None + if args.version: + try: + provided_version = next( + version + for version in UnRPA.provided_versions + if args.version.lower() == version.name.lower() + ) + except StopIteration: + parser.error( + "The archive version you gave isn’t one we recognise - it needs to be one of: " + + ", ".join(version.name for version in UnRPA.provided_versions) + ) -try: - extractor = UnRPA( - args.filename, - args.verbose, - args.path, - args.mkdir, - provided_version, - args.continue_on_error, - provided_offset_and_key, - ) - if args.list: - extractor.list_files() - else: - extractor.extract_files() -except UnRPAError as error: - sys.exit(f"\n\033[31m{error.message}\n{error.cmd_line_help}\033[30m") + provided_offset_and_key: Optional[Tuple[int, int]] = None + if args.key and args.offset: + provided_offset_and_key = (args.offset, args.key) + elif bool(args.key) != bool(args.offset): + parser.error("If you set --key or --offset, you must set both.") + + if args.list and args.path: + parser.error("Option -path: only valid when extracting.") + + if args.mkdir and not args.path: + parser.error("Option --mkdir: only valid when --path is set.") + + if not args.mkdir and args.path and not os.path.isdir(args.path): + parser.error(f"No such directory: “{args.path}”. Use --mkdir to create it.") + + if args.list and args.verbose == 0: + parser.error("Option --list: can’t be silent while listing data.") + + if not os.path.isfile(args.filename): + parser.error(f"No such file: “{args.filename}”.") + + try: + extractor = UnRPA( + args.filename, + args.verbose, + args.path, + args.mkdir, + provided_version, + args.continue_on_error, + provided_offset_and_key, + ) + if args.list: + extractor.list_files() + else: + extractor.extract_files() + except UnRPAError as error: + help_message = f"\n{error.cmd_line_help}" if error.cmd_line_help else "" + sys.exit(f"\n\033[31m{error.message}{help_message}\033[30m") + + +if __name__ == "__main__": + main() diff --git a/unrpa/versions/zix.py b/unrpa/versions/zix.py index 6cb1731..e2d1001 100644 --- a/unrpa/versions/zix.py +++ b/unrpa/versions/zix.py @@ -1,8 +1,9 @@ +import ast import io +import itertools import os import re import struct -import itertools from typing import BinaryIO, Tuple, Optional, FrozenSet, Type from unrpa.versions.errors import ( @@ -12,96 +13,84 @@ from unrpa.versions.version import HeaderBasedVersion, Version from unrpa.view import ArchiveView +loader_name = "loader.pyo" -class ZiX12B(HeaderBasedVersion): + +def get_loader(archive: BinaryIO) -> str: + path = os.path.join(os.path.dirname(archive.name), loader_name) + try: + import uncompyle6 # type: ignore + except ImportError as e: + raise MissingPackageError("uncompyle6") from e + try: + with io.StringIO() as decompiled: + uncompyle6.decompile_file(path, outstream=decompiled) + return decompiled.getvalue() + except ImportError as e: + raise LoaderRequiredError(path) from e + + +def find_key(loader: str) -> int: + vc_match = re.search(r"verificationcode = _string.sha1\('(.*?)'\)", loader) + if not vc_match: + raise IncorrectLoaderError() + else: + return obfuscation_sha1(vc_match.group(1)) + + +def find_offset(archive: BinaryIO) -> int: + return obfuscation_offset(archive.readline().split()[-1]) + + +class ZiX12A(HeaderBasedVersion): """A proprietary format with additional obfuscation.""" - name = "ZiX-12B" - header = b"ZiX-12B" + name = "ZiX-12A" + header = b"ZiX-12A" + + def find_offset_and_key(self, archive: BinaryIO) -> Tuple[int, Optional[int]]: + loader = get_loader(archive) + key = find_key(loader) + return find_offset(archive), key - magic_constant = 102464652121606009 - magic_keys = ( - 3621826839565189698, - 8167163782024462963, - 5643161164948769306, - 4940859562182903807, - 2672489546482320731, - 8917212212349173728, - 7093854916990953299, - ) - loader = "loader.pyo" +class ZiX12B(HeaderBasedVersion): + """A proprietary format with additional obfuscation.""" - struct_format = " None: - self.key: Optional[int] = None + self.details: Optional[Tuple[int, int]] = None def find_offset_and_key(self, archive: BinaryIO) -> Tuple[int, Optional[int]]: - path = os.path.join(os.path.dirname(archive.name), ZiX12B.loader) - try: - import uncompyle6 # type: ignore - except ImportError as e: - raise MissingPackageError("uncompyle6") from e - try: - with io.StringIO() as decompiled: - uncompyle6.decompile_file(path, outstream=decompiled) - match = re.search( - r"verificationcode = _string.sha1\('(.*)'\)", decompiled.getvalue() - ) - if match: - verification_code = match.group(1) - else: - raise IncorrectLoaderError() - except ImportError as e: - raise LoaderRequiredError(path) from e - parts = archive.readline().split() - self.key = ZiX12B.sha1(verification_code) - return ZiX12B.offset(parts[-1]), self.key + loader = get_loader(archive) + key = find_key(loader) + oa_match = re.search( + r"_string.run\(rv.read\(([0-9]*?)\), verificationcode\)", loader + ) + if not oa_match: + raise IncorrectLoaderError() + else: + self.details = (key, ast.literal_eval(oa_match.group(1))) + return find_offset(archive), key def postprocess(self, source: ArchiveView, sink: BinaryIO) -> None: - """Allows postprocessing over the data extracted from the archive.""" - if self.key: + if self.details: + key, amount = self.details parts = [] - amount = ZiX12B.obfuscated_amount while amount > 0: part = source.read(amount) amount -= len(part) parts.append(part) - sink.write(ZiX12B.run(b"".join(parts), self.key)) + sink.write(obfuscation_run(b"".join(parts), key)) else: raise Exception("find_offset_and_key must be called before postprocess") for segment in iter(source.read1, b""): sink.write(segment) - # The following code is reverse engineered from the cython "_string.pyd" file courtesy of omegalink12. - # https://github.com/Lattyware/unrpa/issues/15#issuecomment-485014225 - - @staticmethod - def sha1(code: str) -> int: - a = int("".join(filter(str.isdigit, code))) + ZiX12B.magic_constant - b = round(a ** (1 / 3)) / 23 * 109 - return int(b) - - @staticmethod - def offset(value: bytes) -> int: - a = value[7:5:-1] - b = value[:3] - c = value[5:2:-1] - return int(a + b + c, 16) - - @staticmethod - def run(s: bytes, key: int) -> bytes: - encoded = struct.unpack(ZiX12B.struct_format, s) - decoded = ( - magic_key ^ key ^ part - for (magic_key, part) in zip(itertools.cycle(ZiX12B.magic_keys), encoded) - ) - return struct.pack(ZiX12B.struct_format, *decoded) - -versions: FrozenSet[Type[Version]] = frozenset({ZiX12B}) +versions: FrozenSet[Type[Version]] = frozenset({ZiX12A, ZiX12B}) class LoaderRequiredError(VersionSpecificRequirementUnmetError): @@ -109,16 +98,54 @@ class LoaderRequiredError(VersionSpecificRequirementUnmetError): def __init__(self, path: str) -> None: super().__init__( - f"To extract {ZiX12B.name} archives, the “{ZiX12B.loader}” file is required alongside the archive (we " - f"looked for it at “{path}”). You can find this file in the game you got the archive from, in the “renpy” " - f"directory.", - f"Copy the “{ZiX12B.loader}” file next to the archive you are trying to extract.", + f"To extract ZiX archives, the “{loader_name}” file is required alongside the archive (we looked for it at " + f"“{path}”). You can find this file in the game you got the archive from, in the “renpy” directory.", + f"Copy the “{loader_name}” file next to the archive you are trying to extract.", ) class IncorrectLoaderError(VersionSpecificRequirementUnmetError): def __init__(self) -> None: super().__init__( - "The provided “{ZiX12B.loader}” file does not appear to be the correct one. Please check it is from the " + f"The provided “{loader_name}” file does not appear to be the correct one. Please check it is from the " "game this archive came from." ) + + +# The following code is reverse engineered from the cython "_string.pyd" file courtesy of omegalink12. +# https://github.com/Lattyware/unrpa/issues/15#issuecomment-485014225 +# They are somewhat misleadingly named. + +magic_keys = ( + 3621826839565189698, + 8167163782024462963, + 5643161164948769306, + 4940859562182903807, + 2672489546482320731, + 8917212212349173728, + 7093854916990953299, +) + + +def obfuscation_sha1(code: str) -> int: + a = int("".join(filter(str.isdigit, code))) + 102464652121606009 + b = round(a ** (1 / 3)) / 23 * 109 + return int(b) + + +def obfuscation_offset(value: bytes) -> int: + a = value[7:5:-1] + b = value[:3] + c = value[5:2:-1] + return int(a + b + c, 16) + + +def obfuscation_run(s: bytes, key: int) -> bytes: + count = len(s) // 8 + struct_format = f"<{'Q'*count}" + encoded = struct.unpack(struct_format, s) + decoded = ( + magic_key ^ key ^ part + for (magic_key, part) in zip(itertools.cycle(magic_keys), encoded) + ) + return struct.pack(struct_format, *decoded) diff --git a/unrpa/view.py b/unrpa/view.py index f958cf4..5a17f47 100644 --- a/unrpa/view.py +++ b/unrpa/view.py @@ -1,16 +1,15 @@ import io -from typing import cast, Callable +from typing import cast, Callable, BinaryIO class ArchiveView: """A file-like object that just passes through to the underlying file.""" - def __init__( - self, archive: io.BufferedIOBase, offset: int, length: int, prefix: bytes - ): + def __init__(self, archive: BinaryIO, offset: int, length: int, prefix: bytes): archive.seek(offset) + self.name = archive.name self.remaining = length - self.sources = [archive] + self.sources = [cast(io.BufferedIOBase, archive)] if prefix: self.sources.insert(0, cast(io.BufferedIOBase, io.BytesIO(prefix)))