From 9ceee85aab737ca6596d7f9015a01245cf051de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Fri, 14 Jul 2023 16:24:24 +0200 Subject: [PATCH] Start working on extensions. - Update translations task to put translations under extensions. - Put translations in the right place. - Remove INI bakery plugin. - Use mo2- prefixed extensions for translations. - Generate metadata for the translations extension. --- mob.ini | 6 +- src/CMakeLists.txt | 2 +- src/core/conf.cpp | 41 ++++++--- src/core/conf.h | 40 ++++++--- src/core/op.cpp | 24 ++++++ src/core/op.h | 5 ++ src/main.cpp | 22 +---- src/tasks/modorganizer.cpp | 5 -- src/tasks/sevenz.cpp | 2 +- src/tasks/tasks.h | 17 +--- src/tasks/translations.cpp | 169 +++++++++++++++++++++---------------- 11 files changed, 191 insertions(+), 142 deletions(-) diff --git a/mob.ini b/mob.ini index a7f12fa..d50a279 100644 --- a/mob.ini +++ b/mob.ini @@ -21,6 +21,10 @@ host = super = cmake_common modorganizer* githubpp plugins = check_fnis bsapacker bsa_extractor diagnose_basic installer_* plugin_python preview_base preview_bsa tool_* game_* +[translations] +mo2-translations = organizer +mo2-game-bethesda = game_creation game_enderal game_enderalse game_fallout3 game_fallout4 game_fallout4vr game_falloutNV game_gamebryo game_morrowind game_nehrim game_oblivion game_skyrim game_skyrimse game_skyrimvr game_ttw + [task] enabled = true mo_org = ModOrganizer2 @@ -144,10 +148,10 @@ install_pdbs = install_dlls = install_loot = install_plugins = +install_extensions = install_stylesheets = install_licenses = install_pythoncore = -install_translations = vs = qt_install = qt_bin = diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b87b84b..67237f4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,7 +4,7 @@ file(GLOB_RECURSE source_files *.cpp) file(GLOB_RECURSE header_files *.h) add_executable(mob ${source_files} ${header_files}) -set_target_properties(mob PROPERTIES CXX_STANDARD 20) +set_target_properties(mob PROPERTIES CXX_STANDARD 23) target_compile_definitions(mob PUBLIC NOMINMAX) target_compile_options(mob PUBLIC "/MT") diff --git a/src/core/conf.cpp b/src/core/conf.cpp index acaa076..a9ad3d0 100644 --- a/src/core/conf.cpp +++ b/src/core/conf.cpp @@ -34,24 +34,32 @@ namespace mob::details { // returns a string from conf, bails out if it doesn't exist // - std::string get_string(std::string_view section, std::string_view key) + std::string get_string(std::string_view section, std::string_view key, + std::optional default_) { auto sitor = g_conf.find(section); if (sitor == g_conf.end()) gcx().bail_out(context::conf, "[{}] doesn't exist", section); auto kitor = sitor->second.find(key); - if (kitor == sitor->second.end()) - gcx().bail_out(context::conf, "no key '{}' in [{}]", key, section); + if (kitor == sitor->second.end()) { + if (!default_.has_value()) { + gcx().bail_out(context::conf, "no key '{}' in [{}]", key, section); + } + return *default_; + } return kitor->second; } // calls get_string(), converts to int // - int get_int(std::string_view section, std::string_view key) + int get_int(std::string_view section, std::string_view key, + std::optional default_) { - const auto s = get_string(section, key); + const auto s = get_string(section, key, default_.transform([](auto v) { + return std::to_string(v); + })); try { return std::stoi(s); @@ -63,9 +71,12 @@ namespace mob::details { // calls get_string(), converts to bool // - bool get_bool(std::string_view section, std::string_view key) + bool get_bool(std::string_view section, std::string_view key, + std::optional default_) { - const auto s = get_string(section, key); + const auto s = get_string(section, key, default_.transform([](auto v) { + return v ? "true" : "false"; + })); return bool_from_string(s); } @@ -378,13 +389,8 @@ namespace mob { MOB_ASSERT(!tasks.empty()); - for (auto& t : tasks) { - if (t->name() != task && - details::find_string_for_task(t->name(), key)) { - continue; - } + for (auto& t : tasks) details::set_string_for_task(t->name(), key, value); - } } else { // global task option @@ -488,7 +494,7 @@ namespace mob { resolve_path("install_licenses", p.install_bin(), "licenses"); resolve_path("install_pythoncore", p.install_bin(), "pythoncore"); resolve_path("install_stylesheets", p.install_bin(), "stylesheets"); - resolve_path("install_translations", p.install_bin(), "translations"); + resolve_path("install_extensions", p.install_bin(), "extensions"); // finally, resolve the tools that are unlikely to be in PATH; all the // other tools (7z, jom, patch, etc.) are assumed to be in PATH (which @@ -634,6 +640,11 @@ namespace mob { return {}; } + conf_translations conf::translation() + { + return {}; + } + conf_prebuilt conf::prebuilt() { return {}; @@ -725,6 +736,8 @@ namespace mob { conf_versions::conf_versions() : conf_section("versions") {} + conf_translations::conf_translations() : conf_section("translations") {} + conf_prebuilt::conf_prebuilt() : conf_section("prebuilt") {} conf_paths::conf_paths() : conf_section("paths") {} diff --git a/src/core/conf.h b/src/core/conf.h index 891fc83..fb27dc0 100644 --- a/src/core/conf.h +++ b/src/core/conf.h @@ -7,15 +7,18 @@ namespace mob::details { // returns an option named `key` from the given `section` // - std::string get_string(std::string_view section, std::string_view key); + std::string get_string(std::string_view section, std::string_view key, + std::optional default_ = {}); // calls get_string(), converts to bool // - bool get_bool(std::string_view section, std::string_view key); + bool get_bool(std::string_view section, std::string_view key, + std::optional default_ = {}); // calls get_string(), converts to in // - int get_int(std::string_view section, std::string_view key); + int get_int(std::string_view section, std::string_view key, + std::optional default_ = {}); // sets the given option, bails out if the option doesn't exist // @@ -46,25 +49,31 @@ namespace mob { template class conf_section { public: - DefaultType get(std::string_view key) const + DefaultType get(std::string_view key, + std::optional default_ = {}) const { - return details::get_string(name_, key); + if constexpr (std::is_same_v) { + return details::get_string(name_, key, default_); + } + else { + return details::get_string(name_, key); + } } // undefined template - T get(std::string_view key) const; + T get(std::string_view key, std::optional default_ = {}) const; template <> - bool get(std::string_view key) const + bool get(std::string_view key, std::optional default_) const { - return details::get_bool(name_, key); + return details::get_bool(name_, key, default_); } template <> - int get(std::string_view key) const + int get(std::string_view key, std::optional default_) const { - return details::get_int(name_, key); + return details::get_int(name_, key, default_); } void set(std::string_view key, std::string_view value) @@ -220,6 +229,13 @@ namespace mob { conf_versions(); }; + // options in [translations] + // + class conf_translations : public conf_section { + public: + conf_translations(); + }; + // options in [prebuilt] // class conf_prebuilt : public conf_section { @@ -254,11 +270,10 @@ namespace mob { VALUE(install_dlls); VALUE(install_loot); - VALUE(install_plugins); + VALUE(install_extensions); VALUE(install_stylesheets); VALUE(install_licenses); VALUE(install_pythoncore); - VALUE(install_translations); VALUE(vs); VALUE(qt_install); @@ -282,6 +297,7 @@ namespace mob { conf_cmake cmake(); conf_tools tool(); conf_transifex transifex(); + conf_translations translation(); conf_prebuilt prebuilt(); conf_versions version(); conf_paths path(); diff --git a/src/core/op.cpp b/src/core/op.cpp index bf0a051..62fc7e9 100644 --- a/src/core/op.cpp +++ b/src/core/op.cpp @@ -116,6 +116,30 @@ namespace mob::op { } } + void delete_file_glob_recurse(const context& cx, const fs::path& directory, + const fs::path& glob, flags f) + { + cx.trace(context::fs, "deleting glob {}", glob); + + const auto native = glob.native(); + + if (!fs::exists(directory)) + return; + + for (auto&& e : fs::recursive_directory_iterator(directory)) { + const auto p = e.path(); + const auto name = p.filename().native(); + + if (!PathMatchSpecW(name.c_str(), native.c_str())) { + cx.trace(context::fs, "{} did not match {}; skipping", name, glob); + + continue; + } + + delete_file(cx, p, f); + } + } + void remove_readonly(const context& cx, const fs::path& dir, flags f) { cx.trace(context::fs, "removing read-only from {}", dir); diff --git a/src/core/op.h b/src/core/op.h index eafd58a..15d0a4b 100644 --- a/src/core/op.h +++ b/src/core/op.h @@ -65,6 +65,11 @@ namespace mob::op { // void delete_file_glob(const context& cx, const fs::path& glob, flags f = noflags); + // deletes all files matching the glob in the given directory and its subdirectories + // + void delete_file_glob_recurse(const context& cx, const fs::path& directory, + const fs::path& glob, flags f = noflags); + // removes the readonly flag for all files in `dir`, recursive // void remove_readonly(const context& cx, const fs::path& dir, flags f = noflags); diff --git a/src/main.cpp b/src/main.cpp index a260975..0682e66 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -74,33 +74,13 @@ namespace mob { .add_task("modorganizer-nxmhandler") .add_task("modorganizer-helper") .add_task("githubpp") - .add_task("modorganizer-game_gamebryo") .add_task({"modorganizer-bsapacker", "bsa_packer"}) .add_task("modorganizer-preview_bsa"); - // the gamebryo flag must be set for all game plugins that inherit from - // the gamebryo classes; this will merge the .ts file from gamebryo with - // the one from the specific plugin - add_task() - .add_task("modorganizer-game_oblivion", mo::gamebryo) - .add_task("modorganizer-game_nehrim", mo::gamebryo) - .add_task("modorganizer-game_fallout3", mo::gamebryo) - .add_task("modorganizer-game_fallout4", mo::gamebryo) - .add_task("modorganizer-game_fallout4vr", mo::gamebryo) - .add_task("modorganizer-game_fallout76", mo::gamebryo) - .add_task("modorganizer-game_falloutnv", mo::gamebryo) - .add_task("modorganizer-game_morrowind", mo::gamebryo) - .add_task("modorganizer-game_skyrim", mo::gamebryo) - .add_task("modorganizer-game_skyrimse", mo::gamebryo) - .add_task("modorganizer-game_skyrimvr", mo::gamebryo) - .add_task("modorganizer-game_starfield", mo::gamebryo) - .add_task("modorganizer-game_ttw", mo::gamebryo) - .add_task("modorganizer-game_enderal", mo::gamebryo) - .add_task("modorganizer-game_enderalse", mo::gamebryo); + add_task().add_task("modorganizer-game_bethesda"); add_task() .add_task({"modorganizer-tool_inieditor", "inieditor"}) - .add_task({"modorganizer-tool_inibakery", "inibakery"}) .add_task("modorganizer-preview_base") .add_task("modorganizer-diagnose_basic") .add_task("modorganizer-check_fnis") diff --git a/src/tasks/modorganizer.cpp b/src/tasks/modorganizer.cpp index 686f905..7f1e2ba 100644 --- a/src/tasks/modorganizer.cpp +++ b/src/tasks/modorganizer.cpp @@ -80,11 +80,6 @@ namespace mob::tasks { } } - bool modorganizer::is_gamebryo_plugin() const - { - return is_set(flags_, gamebryo); - } - bool modorganizer::is_nuget_plugin() const { return is_set(flags_, nuget); diff --git a/src/tasks/sevenz.cpp b/src/tasks/sevenz.cpp index f552508..1a107dd 100644 --- a/src/tasks/sevenz.cpp +++ b/src/tasks/sevenz.cpp @@ -79,7 +79,7 @@ namespace mob::tasks { void sevenz::build() { - build_loop(cx(), [&](bool mp) { + build_loop(cx(), [&]([[maybe_unused]] bool mp) { const int exit_code = run_tool(nmake() .path(module_to_build()) .def("CPU=x64") diff --git a/src/tasks/tasks.h b/src/tasks/tasks.h index bc24139..b742ef5 100644 --- a/src/tasks/tasks.h +++ b/src/tasks/tasks.h @@ -235,13 +235,9 @@ namespace mob::tasks { enum flags { noflags = 0x00, - // gamebryo project, used by the translations task because these - // projects have multiple .ts files that have to be merged - gamebryo = 0x01, - // project that uses nuget, cmake doesn't support those right now, so // `msbuild -t:restore` has to be run manually - nuget = 0x02, + nuget = 0x01, }; // some mo tasks have more than one name, mostly because the transifex slugs @@ -252,10 +248,6 @@ namespace mob::tasks { modorganizer(std::vector names, flags f = noflags); modorganizer(std::vector names, flags f = noflags); - // whether this project has the gamebryo flag on - // - bool is_gamebryo_plugin() const; - // whether this project has the nuget flag on // bool is_nuget_plugin() const; @@ -632,11 +624,6 @@ namespace mob::tasks { // duplicate warnings std::set warned_; - // whether the given project name is a gamebryo task, `dir` is just for - // logging - // - bool is_gamebryo_plugin(const std::string& dir, const std::string& project); - // parses the directory name, walks all the .ts files, returns a project // object for them // @@ -645,7 +632,7 @@ namespace mob::tasks { // returns a lang object that contains at least the given main_ts_file, // but might contain more if it's a gamebryo plugin // - lang create_lang(bool gamebryo, const std::string& project_name, + lang create_lang(const std::string& project_name, const fs::path& main_ts_file); }; diff --git a/src/tasks/translations.cpp b/src/tasks/translations.cpp index dac500a..957f6cb 100644 --- a/src/tasks/translations.cpp +++ b/src/tasks/translations.cpp @@ -1,6 +1,8 @@ #include "pch.h" #include "../core/env.h" +#include "../utility/string.h" #include "../utility/threading.h" +#include "nlohmann/json.hpp" #include "task_manager.h" #include "tasks.h" @@ -150,7 +152,6 @@ namespace mob::tasks { // project project p(project_name); - const bool gamebryo = is_gamebryo_plugin(dir_name, project_name); // for each file for (auto f : fs::directory_iterator(dir)) { @@ -168,14 +169,14 @@ namespace mob::tasks { } // add a new `lang` object for it - p.langs.push_back(create_lang(gamebryo, project_name, f.path())); + p.langs.push_back(create_lang(project_name, f.path())); } return p; } translations::projects::lang - translations::projects::create_lang(bool gamebryo, const std::string& project_name, + translations::projects::create_lang(const std::string& project_name, const fs::path& main_ts_file) { lang lg(path_to_utf8(main_ts_file.stem())); @@ -183,64 +184,9 @@ namespace mob::tasks { // every lang has the .ts file from the project, gamebryo plugins have more lg.ts_files.push_back(main_ts_file); - if (gamebryo) { - // this is a gamebryo plugin, so it needs the gamebryo .ts file as well, - // find it - - // the .ts files for gamebryo are in mod-organizer-2.game_gamebryo/ - const fs::path gamebryo_dir = - conf().transifex().get("project") + "." + "game_gamebryo"; - - // the .ts file has the same name, it's just "lang.ts" - const auto gamebryo_ts = root_ / gamebryo_dir / main_ts_file.filename(); - - if (fs::exists(gamebryo_ts)) { - // found, add it - lg.ts_files.push_back(gamebryo_ts); - } - else { - // not found, that means the plugin was translated into a language, - // but the gamebryo project wasn't; warn once - if (!warned_.contains(gamebryo_ts)) { - warned_.insert(gamebryo_ts); - - warnings_.push_back(::fmt::format( - "{} is a gamebryo plugin but there is no '{}'; the " - ".qm file will be missing some translations (will " - "only warn once)", - project_name, path_to_utf8(gamebryo_ts))); - } - } - } - return lg; } - bool translations::projects::is_gamebryo_plugin(const std::string& dir, - const std::string& project) - { - const auto* t = task_manager::instance().find_one(project); - - if (!t) { - warnings_.push_back( - ::fmt::format("directory '{}' was parsed as project '{}', but there's " - "no task with this name", - dir, project)); - - return false; - } - - // gamebryo plugins are all `modorganizer` tasks - const auto* mo_task = static_cast(t); - if (!mo_task) { - // not an mo task, can't be a gamebryo plugin - return false; - } - - // check the flag - return mo_task->is_gamebryo_plugin(); - } - translations::translations() : task("translations") {} fs::path translations::source_path() @@ -256,8 +202,8 @@ namespace mob::tasks { // remove the .qm files in the translations/ directory if (is_set(c, clean::rebuild)) { - op::delete_file_glob(cx(), conf().path().install_translations() / "*.qm", - op::optional); + op::delete_file_glob_recurse(cx(), conf().path().install_extensions(), + "*.qm", op::optional); } } @@ -310,44 +256,122 @@ namespace mob::tasks { } } + void generate_translations_metadata( + std::filesystem::path const& path, + std::vector const& languages) + { + using json = nlohmann::ordered_json; + + json metadata; + { + std::ifstream ifs(path); + metadata = json::parse(ifs); + } + + // fix version + json translations; + for (auto&& lang : languages) { + std::vector files; + files.push_back("translations/" + lang.name + "/*.qm"); + json jsonlang; + jsonlang["files"] = files; + translations[lang.name] = jsonlang; + } + metadata["content"]["translations"] = translations; + + std::ofstream ofs(path); + ofs << metadata.dump(2); + } + void translations::do_build_and_install() { // 1) build the list of projects, languages and .ts files // 2) run `lrelease` for every language in every project // 3) copy builtin qt translations - const auto root = source_path() / "translations"; - const auto dest = conf().path().install_translations(); + const auto root = source_path() / "translations"; + const auto extensions = conf().path().install_extensions(); const projects ps(root); - op::create_directories(cx(), dest); + op::create_directories(cx(), extensions / "mo2-translations"); // log all the warnings added while walking the projects for (auto&& w : ps.warnings()) cx().warning(context::generic, "{}", w); + std::map project_to_extension; + + // go through the list of extensions and find the matching projects + for (auto& p : fs::directory_iterator(extensions)) { + if (!fs::is_directory(p)) + continue; + + const auto name = + mob::replace_all(p.path().filename().string(), "mo2-", ""); + + project_to_extension[name] = p.path().filename(); + project_to_extension[mob::replace_all(name, "-", "_")] = + p.path().filename(); + + const auto s_projects = + conf().translation().get(p.path().filename().string(), ""); + if (!s_projects.empty()) { + const auto extension_projects = mob::split(s_projects, " "); + for (const auto& project : extension_projects) { + project_to_extension[project] = p.path().filename(); + } + } + } + // run `lrelease` in a thread pool parallel_functions v; // for each project for (auto& p : ps.get()) { + if (!project_to_extension.contains(p.name)) { + cx().warning(context::generic, + "found project {} but no matching extension", p.name); + continue; + } + + const auto base = extensions / project_to_extension[p.name]; + + if (!fs::exists(base)) { + cx().warning(context::generic, + "found project {} for extension {} extension is not built", + p.name, project_to_extension[p.name]); + continue; + } + + const auto dest = base / "translations"; + op::create_directories(cx(), dest); + // for each language for (auto& lg : p.langs) { + op::create_directories(cx(), dest / lg.name); // add a functor that will run lrelease - v.push_back( - {lg.name + "." + p.name, [&] { - // run release for the given project name and list of .ts files - run_tool( - lrelease().project(p.name).sources(lg.ts_files).out(dest)); - }}); + v.push_back({lg.name + "." + p.name, [=] { + // run release for the given project name and list of + // .ts files + run_tool(lrelease() + .project(p.name) + .sources(lg.ts_files) + .out(dest / lg.name)); + }}); } } // run all the functors in parallel parallel(v); - if (auto p = ps.find("organizer")) - copy_builtin_qt_translations(*p, dest); + if (auto p = ps.find("organizer")) { + // the empty metadata for mo2-translations is copied from modorganizer and + // then filled here + generate_translations_metadata( + extensions / "mo2-translations" / "metadata.json", p->langs); + copy_builtin_qt_translations(*p, extensions / "mo2-translations" / + "translations"); + } else cx().bail_out(context::generic, "organizer project not found"); } @@ -366,7 +390,8 @@ namespace mob::tasks { if (!fs::exists(src)) return false; - op::copy_file_to_dir_if_better(cx(), src, dest, op::unsafe); + op::create_directories(cx(), dest / lang); + op::copy_file_to_dir_if_better(cx(), src, dest / lang, op::unsafe); return true; };