diff --git a/interface_gen/generate.py b/interface_gen/generate.py index 4dae634..f2320a6 100644 --- a/interface_gen/generate.py +++ b/interface_gen/generate.py @@ -46,7 +46,8 @@ def find_in_path(self, path: str | None = None) -> 'Schemas': self.schemas = list(start_path.rglob("*.avsc")) return self - def from_avro_idl(self, output_dir: Path) -> 'Schemas': + def from_avro_idl(self, env: dict[str, str], avro_cmd: list[str], + output_dir: Path) -> 'Schemas': """ Generate all schemas from main Avro IDL files, and select all schemas for future operations on this object. """ protocol_path = self.proto_dir @@ -55,9 +56,8 @@ def from_avro_idl(self, output_dir: Path) -> 'Schemas': print(f"--> Generating schema(s) for {avdl}") version_dir = avdl.parent.name sdir = output_dir / version_dir / "schema" - script = toolchain.script_dir().parent / "avro" / "bin" / "avro-idl.sh" - subprocess.check_call([str(script), str(avdl), str(sdir)]) - return self.find_in_path(output_dir) + subprocess.check_call(avro_cmd + [str(avdl), str(sdir)], env=env) + return self.find_in_path(str(output_dir)) # --> Action methods @@ -68,8 +68,9 @@ def gen_proto3(self): pb_dir.mkdir(parents=True, exist_ok=True) print(f"--> Generating proto3 for {str(schema_file)} in {pb_dir}") convert_avro_to_proto(schema_file, pb_dir) - # workaround: avrotize func. above always names file .proto, - # which causes all except the last schema to be overwritten. Rename that + # workaround: avrotize func. above always names file + # .proto, which causes all except the last schema to be + # overwritten. Rename that # output file here, until we can fix the avrotize lib. ns = namespace(schema_file) if ns: @@ -87,20 +88,19 @@ def main(): Output will be written to the output directory (-o): Avro schemas, proto3 definitions, markdown documentation. """ - parser = argparse.ArgumentParser(prog="generate.py", - description="Generate docs and definitions from Avro IDL.", + desc = "Generate docs and definitions from Avro IDL." + parser = argparse.ArgumentParser(prog="generate.py", description=desc, epilog=epilog) parser.add_argument('-p', '--protocol-dir', help="Path w/ Avro IDL files in version subdirs") parser.add_argument('-o', '--output-dir', required=True, help="Path where to generate markdown docs") parser.add_argument('-i', '--install-toolchain', - help="Install toolchain dependencies if needed.", + help="(deprecated: always attempts toolchain install)", action="store_true") args = parser.parse_args() - if args.install_toolchain: - toolchain.install() + (env, avro_cmd) = toolchain.install() if args.protocol_dir: proto_dir = Path(args.protocol_dir) @@ -113,7 +113,7 @@ def main(): out_dir = proto_dir.parent / "generated" print("--> Generating Avro schemas..") - schemas = Schemas(proto_dir).from_avro_idl(out_dir) + schemas = Schemas(proto_dir).from_avro_idl(env, avro_cmd, out_dir) print(f"--> Found schemas: {schemas.schemas}") print("--> Generating proto3 definitions for all schemas") diff --git a/interface_gen/toolchain.py b/interface_gen/toolchain.py index 9b94238..7956f72 100644 --- a/interface_gen/toolchain.py +++ b/interface_gen/toolchain.py @@ -1,4 +1,5 @@ import jdk +import os from pathlib import Path import subprocess import shutil @@ -20,17 +21,34 @@ def avro_jar(version: str) -> Path: return Path(f"avro-tools-{version}.jar") -def ensure_jre(): - """ Ensure that JRE is installed. """ +def find_java(root: Path) -> Path | None: + java_files = list(root.rglob("java")) + if len(java_files) == 0: + print(f"Java not found in {str(root)}") + return None + print(f"found java: {list(map(lambda p: str(p), java_files))}") + java_path = Path(java_files[0]) + env = java_env(java_path) + subprocess.check_output([str(java_path), '-version'], env=env) + return java_path + + +def ensure_jre() -> Path | None: + """ Ensure that JRE is installed, and return its path. """ + jre_dir = Path.home() / '.jre' # See if jdk / jre already installedd jre = shutil.which('java') + if not jre: + jre = find_java(jre_dir) if jre: print(f"(Java is already installed: {jre})") - return + return Path(jre) - print("--> Installing OpenJDK 22 in $HOME/.jre/") + print(f"--> Installing OpenJDK 22 in {jre_dir}") jdk.install('22', jre=True) + return find_java(jre_dir) + def install_avro_tools(target_dir: Path): global AVRO_VERSION @@ -44,25 +62,36 @@ def install_avro_tools(target_dir: Path): download_file(url, target_dir) -def create_avro_script(target_dir: Path): - target_dir.mkdir(parents=True, exist_ok=True) - script = target_dir / "avro-idl.sh" - print(f"--> Generating IDL->Schema script {script}") - with script.open("w") as f: - f.write(f"""\ -#!/bin/bash +def java_env(java_path: Path) -> dict[str, str]: + """ Return environment with path, etc. set for java command """ + bin_dir = java_path.parent + java_home = bin_dir.parent + lib_dir = java_home / 'lib' + new_path = os.environ['PATH'] + f":{bin_dir}" + env = os.environ.copy() + env['PATH'] = new_path + env['LD_LIBRARY_PATH'] = str(lib_dir) + return env + -# This script is generated by toolchain.py -java -jar {target_dir}/{avro_jar(AVRO_VERSION)} idl2schemata "$@" -""") - script.chmod(0o755) +# TODO make this file a class that retains toolchain paths, etc. and provides a +# method to run the command +def create_avro_cmd(java_path: Path, target_dir: Path) \ + -> tuple[dict[str, str], list[str]]: + env = java_env(java_path) + return (env, [str(java_path), '-jar', + f'{target_dir}/{avro_jar(AVRO_VERSION)}', + 'idl2schemata']) -def install(): - """ Install the toolchain: tools needed for build and code generation. """ +def install() -> tuple[dict[str, str], list[str]]: + """ Install the toolchain and return the command needed to generate + schemas and code. """ sdir = script_dir() avro_bin = sdir.parent / "avro" / "bin" - ensure_jre() + java_path = ensure_jre() + if not java_path: + raise Exception("Failed to install java.") install_avro_tools(avro_bin) - create_avro_script(avro_bin) + return create_avro_cmd(java_path, avro_bin) diff --git a/pyproject.toml b/pyproject.toml index 9ecb6ad..0781a0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "interface_gen" -version = "0.0.19" +version = "0.0.20" authors = [{ name = "Aaron Fabbri", email = "aaron.fabbri@ditto.live" }] description = "Define data format schemas once and generate code for multiple languages and serialization frameworks." readme = "dist-readme.md"