configure.py

#! /usr/bin/env python3

import argparse
import json
import os
import shutil
import subprocess
import sys
import re

from pathlib import Path
from typing import Any, Dict, List, Set, Union, Optional

import ninja_syntax

import splat
import splat.scripts.split as split
from splat.segtypes.linker_entry import LinkerEntry

ROOT = Path(__file__).parent
TOOLS_DIR = ROOT / "tools"

YAML_FILE = "config/xenosaga.yaml"
BASENAME = "SLUS_204.69"
LD_PATH = f"{BASENAME}.ld"
ELF_PATH = f"build/{BASENAME}"
MAP_PATH = f"build/{BASENAME}.map"
PRE_ELF_PATH = f"build/{BASENAME}.elf"

COMMON_INCLUDES = "-Iinclude -Isrc -isystem include/sdk/ee -isystem include/sdk -isystem include/gcc -isystem include/gcc/gcc-lib"
COMPILER_DIR = f"{TOOLS_DIR}/cc/ee-gcc2.96/bin"

COMPILER_FLAGS = "-O2 -Wa,-Iinclude"
COMPILER_FLAGS_CPP = "-O2 c++"

COMPILE_CMD = f"{COMPILER_DIR}/ee-gcc -c {COMMON_INCLUDES} {COMPILER_FLAGS}"
COMPILE_CMD_CPP = f"{COMPILER_DIR}/ee-gcc -c {COMMON_INCLUDES} {COMPILER_FLAGS_CPP}"

WIBO_VER = "0.6.11"

# CALCULATE PROGRESS TODO:
# python3 -m mapfile_parser progress build/SCPS_150.17.map asm asm/nonmatchings/


def exec_shell(command: List[str], stdout=subprocess.PIPE) -> str:
    ret = subprocess.run(command, stdout=stdout, stderr=subprocess.PIPE, text=True)
    return ret.stdout


def clean():
    if os.path.exists(".splache"):
        os.remove(".splache")
    if os.path.exists("compile_commands.json"):
        os.remove("compile_commands.json")
    shutil.rmtree("asm", ignore_errors=True)
    shutil.rmtree("assets", ignore_errors=True)
    shutil.rmtree("build", ignore_errors=True)


gp_access_pattern = re.compile(
    r"%(gp_rel)\(([^)]+)\)\(\$28\)"
)  # Pattern for removing %gp_rel accesses
gp_add_pattern = re.compile(
    r"addiu\s+(\$\d+), \$28, %gp_rel\(([^)]+)\)"
)  # Pattern for replacing %gp_rel additions


def remove_gprel():
    for root, dirs, files in os.walk("asm/nonmatchings/"):
        for filename in files:
            filepath = os.path.join(root, filename)

            with open(filepath, "r") as file:
                content = file.read()

            # Search for any %gp_rel access
            # INSTR REG, %gp_rel(SYMBOL)($28) -> INSTR REG, SYMBOL
            if re.search(gp_access_pattern, content):
                # Reference found, remove
                content = re.sub(gp_access_pattern, r"\2", content)

                # Write the updated content back to the file
                with open(filepath, "w") as file:
                    file.write(content)

            # Search for any %gp_rel additions
            # addiu REG, $28, %gp_rel(SYMBOL) -> la REG, SYMBOL
            if re.search(gp_add_pattern, content):
                # Reference found, replace
                content = re.sub(gp_add_pattern, r"la \1, \2", content)

                # Write the updated content back to the file
                with open(filepath, "w") as file:
                    file.write(content)


def write_permuter_settings():
    with open("permuter_settings.toml", "w") as f:
        f.write(
            f"""compiler_command = "tools/cc/ee-gcc2.96/bin/ee-gcc -c -Iinclude -Iinclude/sdk/ee -Iinclude/gcc -Iinclude/gcc/gcc-lib -O2 -G8 -gstabs -D__GNUC__"
assembler_command = "mips-linux-gnu-as -march=r5900 -mabi=eabi -Iinclude"
compiler_type = "gcc"

[preserve_macros]

[decompme.compilers]
"tools/cc/ee-gcc2.96/bin/ee-gcc" = "ee-gcc2.9-991111-01"
"""
        )


def build_stuff(linker_entries: List[LinkerEntry]):
    built_objects: Set[Path] = set()

    def build(
        object_paths: Union[Path, List[Path]],
        src_paths: List[Path],
        task: str,
        variables: Dict[str, str] = {},
        implicit_outputs: List[str] = [],
    ):
        if not isinstance(object_paths, list):
            object_paths = [object_paths]

        object_strs = [str(obj) for obj in object_paths]

        for object_path in object_paths:
            if object_path.suffix == ".o":
                built_objects.add(object_path)
            ninja.build(
                outputs=object_strs,
                rule=task,
                inputs=[str(s) for s in src_paths],
                variables=variables,
                implicit_outputs=implicit_outputs,
            )

    ninja = ninja_syntax.Writer(open(str(ROOT / "build.ninja"), "w"), width=9999)

    # Rules
    cross = "mips-linux-gnu-"
    ld_args = f"-EL -T config/undefined_syms_auto.txt -T config/undefined_funcs_auto.txt -T config/undefined_syms.txt -Map $mapfile -T $in -o $out"

    ninja.rule(
        "as",
        description="as $in",
        command=f"cpp {COMMON_INCLUDES} $in -o - | iconv -f=UTF-8 -t=EUC-JP $in | {cross}as -no-pad-sections -EL -march=5900 -mabi=eabi -Iinclude -o $out && python3 tools/buildtools/elf_patcher.py $out gas",
    )

    ninja.rule(
        "cpp",
        description="cpp $in",
        command=f"{COMPILE_CMD_CPP} $in -o $out && {cross}strip $out -N dummy-symbol-name",
    )

    ninja.rule(
        "cc",
        description="cc $in",
        command=f"{COMPILE_CMD} $in -o $out && {cross}strip $out -N dummy-symbol-name",
    )

    ninja.rule(
        "ld",
        description="link $out",
        command=f"{cross}ld {ld_args}",
    )

    ninja.rule(
        "sha1sum",
        description="sha1sum $in",
        command="sha1sum -c $in && touch $out",
    )

    ninja.rule(
        "elf",
        description="elf $out",
        command=f"{cross}objcopy $in $out -O binary",
    )

    decompctx = Path("tools") / "decompctx.py"
    ninja.rule(
        name="decompctx",
        command=f"$python {decompctx} $in -o $out -d $out.d",
        description="CTX $in",
        depfile="$out.d",
        deps="gcc",
    )

    for entry in linker_entries:
        seg = entry.segment

        if seg.type[0] == ".":
            continue

        if entry.object_path is None:
            continue

        if isinstance(seg, splat.segtypes.common.asm.CommonSegAsm) or isinstance(
            seg, splat.segtypes.common.data.CommonSegData
        ):
            build(entry.object_path, entry.src_paths, "as")
        elif isinstance(seg, splat.segtypes.common.cpp.CommonSegCpp):
            build(entry.object_path, entry.src_paths, "cpp")
            build(
                entry.object_path.parent / f"{seg.name}.ctx",
                entry.src_paths,
                "decompctx",
            )
        elif isinstance(seg, splat.segtypes.common.c.CommonSegC):
            build(entry.object_path, entry.src_paths, "cc")
            build(
                entry.object_path.parent / f"{seg.name}.ctx",
                entry.src_paths,
                "decompctx",
            )
        elif isinstance(
            seg, splat.segtypes.common.databin.CommonSegDatabin
        ) or isinstance(seg, splat.segtypes.common.rodatabin.CommonSegRodatabin):
            build(entry.object_path, entry.src_paths, "as")
        else:
            print(f"ERROR: Unsupported build segment type {seg.type}")
            sys.exit(1)

    ninja.build(
        PRE_ELF_PATH,
        "ld",
        LD_PATH,
        implicit=[str(obj) for obj in built_objects],
        variables={"mapfile": MAP_PATH},
    )

    ninja.build(
        ELF_PATH,
        "elf",
        PRE_ELF_PATH,
    )

    ninja.build(
        ELF_PATH + ".ok",
        "sha1sum",
        "checksum.sha1",
        implicit=[ELF_PATH],
    )


# Generate objdiff.json
# Hacked together from dtk template. Not generically transferrable to other projects
def generate_objdiff_config(
    linker_entries: List[LinkerEntry],
) -> None:
    objdiff_config: Dict[str, Any] = {
        "min_version": "1.0.0",
        "custom_make": "ninja",
        "build_target": False,
        "watch_patterns": [
            "*.c",
            "*.cp",
            "*.cpp",
            "*.h",
            "*.hpp",
            "*.inc",
            "*.py",
            "*.yml",
            "*.txt",
            "*.json",
        ],
        "units": [],
    }

    build_path = "build"

    def add_unit(entry: LinkerEntry) -> None:
        if entry.segment.type not in ["c", "cpp"]:
            return

        obj_name = entry.segment.name
        unit_config: Dict[str, Any] = {
            "name": entry.segment.name,
            "target_path": Path("expected")
            / entry.object_path.relative_to(Path(build_path)),
        }

        unit_src_path = entry.segment.out_path()

        if not unit_src_path.exists():
            objdiff_config["units"].append(unit_config)
            return

        src_obj_path = entry.object_path
        src_ctx_path = entry.object_path.parent / f"{entry.segment.name}.ctx"

        unit_config["base_path"] = src_obj_path
        # TODO: Detect fully matching TUs
        unit_config["complete"] = not os.path.isdir(
            f"asm/nonmatchings/{entry.segment.name}"
        )
        compiler_version = "ee-gcc2.96"
        unit_config["scratch"] = {
            "platform": "ps2",
            "compiler": compiler_version,
            "c_flags": COMPILER_FLAGS,
            "ctx_path": src_ctx_path,
            "build_ctx": True,
        }
        objdiff_config["units"].append(unit_config)

    # Add units
    for unit in linker_entries:
        add_unit(unit)

    # Write objdiff.json
    with open("objdiff.json", "w", encoding="utf-8") as w:

        def unix_path(input: Any) -> str:
            return str(input).replace(os.sep, "/") if input else ""

        json.dump(objdiff_config, w, indent=4, default=unix_path)


# Workaround hack to avoid the "short loop bug" inserting undesirable noops in some functions.
# replaces all asm instructions in each file in the `short_loop_funcs` array with the raw opcode bytes
opcode_pattern = re.compile(
    r"\/\* [0-9A-Z]+ [0-9A-Z]+ ([0-9A-Z]{2})([0-9A-Z]{2})([0-9A-Z]{2})([0-9A-Z]{2}) .*"
)  # Pattern for removing %gp_rel accesses
nop_bugged_funcs = set(
    [
        "Judge_MakeNewSavedata.s",
        "xglRenderSyncMove.s",
        "AlphaGroupLastEntry.s",
        "nmlModelCalcEntryPartsClip.s",
        "nmlModelEntry.s",
        "nmlModelRenderTexture.s",
        "nmlModelRenderCircle.s",
        "nmlModelRenderDrop.s",
        "nmlModelFlushSubNonAlpha.s",
        "nmlModelFlushSubAlpha.s",
        "nmlModelFlush.s",
        "nmlPacketAddParts.s",
        "nmlFilterSetVolumeCubeRender.s",
        "xglJpegEncode.s",
        "FFIDCT.s",
        "ConvertYUV2RGB.s",
        "Game_Data_Pop.s",
        "checkItemBox.s",
        "GameModeCfMain.s",
        "command_loadsmd.s",
        "xtxdec_sub.s",
        "tyaSiPicOutput.s",
        "STRING_toUInt.s",
        "TWIN_create2.s",
        "updateLight.s",
        "ShopTest.s",
        "OpenSubType01.s",
    ]
)


def replace_instructions_with_opcodes() -> None:
    for root, dirs, files in os.walk("asm/nonmatchings/"):
        for filename in files:
            if filename not in nop_bugged_funcs:
                continue

            filepath = os.path.join(root, filename)

            with open(filepath, "r") as file:
                content = file.read()

            if re.search(opcode_pattern, content):
                # Reference found, replace
                # Embed the opcode, we have to swap byte order for correct endianness
                content = re.sub(opcode_pattern, r".long 0x\4\3\2\1", content)

                # Write the updated content back to the file
                with open(filepath, "w") as file:
                    file.write(content)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Configure the project")
    parser.add_argument(
        "-c",
        "--clean",
        help="Clean extraction and build artifacts",
        action="store_true",
    )
    parser.add_argument(
        "-csrc",
        "--cleansrc",
        help="Clean the 'src' folder",
        action="store_true",
    )
    parser.add_argument(
        "-nogp",
        "--no-gprel-removing",
        help="Do not remove gp_rel references on the disassembly",
        action="store_true",
    )
    parser.add_argument(
        "-noop",
        "--no-opcode-hack",
        help="Do not replace asm instructions with raw opcodes for functions that trigger the short loop bug",
        action="store_true",
    )
    args = parser.parse_args()

    if args.clean:
        clean()

    if args.cleansrc:
        shutil.rmtree("src", ignore_errors=True)

    split.main([YAML_FILE], modes="all", verbose=False)

    linker_entries = split.linker_writer.entries

    build_stuff(linker_entries)

    generate_objdiff_config(linker_entries)

    write_permuter_settings()

    # We're done with everything, now get rid of the %gp_rel references
    if not args.no_gprel_removing:
        remove_gprel()

    if not args.no_opcode_hack:
        replace_instructions_with_opcodes()

    if not os.path.isfile("compile_commands.json"):
        exec_shell(["ninja", "-t", "compdb"], open("compile_commands.json", "w"))