From ecd210d7000af55b431f24941659af9769466d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 15 Jun 2024 22:19:35 +0200 Subject: [PATCH] More flexibility to build Debug components (#136) * Build components in all variant for boost to avoid issue with USVFS. * Build googletest in both Debug and Release. * Allow build configurations other than RelWithDebInfo. --- mob.ini | 21 ++++--- readme.md | 9 +-- src/core/conf.cpp | 118 ++++++++++++++++++++++++++----------- src/core/conf.h | 110 +++++++++++++++++----------------- src/tasks/boost.cpp | 25 ++++---- src/tasks/gtest.cpp | 48 ++++++++++----- src/tasks/libbsarch.cpp | 9 ++- src/tasks/modorganizer.cpp | 52 ++++++++-------- src/tasks/pyqt.cpp | 71 ++++++++++++++++++---- src/tasks/python.cpp | 41 +++++++++---- src/tasks/sip.cpp | 11 +++- src/tasks/tasks.h | 13 +++- src/tasks/usvfs.cpp | 9 ++- src/tools/cmake.cpp | 2 +- src/tools/cmake.h | 6 +- src/tools/msbuild.cpp | 24 ++++++-- src/tools/msbuild.h | 16 +++-- src/tools/python.cpp | 22 ++++++- src/tools/tools.h | 7 +++ src/utility.h | 2 + 20 files changed, 423 insertions(+), 193 deletions(-) diff --git a/mob.ini b/mob.ini index 29160ce..23b2cd6 100644 --- a/mob.ini +++ b/mob.ini @@ -22,13 +22,14 @@ super = cmake_common modorganizer* githubpp plugins = check_fnis bsapacker bsa_extractor diagnose_basic installer_* plugin_python preview_base preview_bsa tool_* game_* [task] -enabled = true -mo_org = ModOrganizer2 -mo_branch = master -mo_fallback = -no_pull = false -ignore_ts = false -revert_ts = false +enabled = true +mo_org = ModOrganizer2 +mo_branch = master +mo_fallback = +no_pull = false +ignore_ts = false +revert_ts = false +configuration = RelWithDebInfo git_url_prefix = https://github.com/ git_shallow = true @@ -118,7 +119,6 @@ zlib = v1.3.1 libbsarch = 0.0.9 usvfs = master explorerpp = 1.4.0 - ss_paper_lad_6788 = 7.2 ss_paper_automata_6788 = 3.2 ss_paper_mono_6788 = 3.2 @@ -129,6 +129,11 @@ ss_starfield_trosski = V1.11 ss_fallout3_trosski = v1.11 ss_fallout4_trosski = v1.11 +[build-types] +libbsarch = release +pyqt = release +python = release + [paths] third_party = prefix = diff --git a/readme.md b/readme.md index ffc9cb0..99ecc0f 100644 --- a/readme.md +++ b/readme.md @@ -68,7 +68,7 @@ aqt install-qt --outputdir "C:\Qt" windows desktop 6.7.0 win64_msvc2019_64 -m qt - Optional: - Qt Source Files - Qt Debug Files - + ### Visual Studio - Install Visual Studio 2022 ([Installer](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=Community&channel=Release&version=VS2022&source=VSLandingPage&cid=2030&passive=false)) - Desktop development with C++ @@ -141,9 +141,10 @@ Inside the INI file are `[sections]` and `key = value` pairs. The `[task]` secti ### `[task]` Options for individual tasks. Can be `[task_name:task]`, where `task_name` is the name of a task (see `mob list`) , `super` for all MO tasks or a glob like `installer_*`. -| Option | Type | Description | -| --- | --- | --- | -| `enabled` | bool | Whether this task is enabled. Disabled tasks are never built. When specifying task names with `mob build task1 task2...`, all tasks except those given are turned off. | +| Option | Type | Description | +| --- | --- | --- | +| `enabled` | bool | Whether this task is enabled. Disabled tasks are never built. When specifying task names with `mob build task1 task2...`, all tasks except those given are turned off. | +| `configuration` | enum | Which configuration to build, should be one of Debug, Release or RelWithDebInfo with RelWithDebInfo being the default.| #### Common git options Unless otherwise stated, applies to any task that is a git repo. diff --git a/src/core/conf.cpp b/src/core/conf.cpp index acaa076..e122f13 100644 --- a/src/core/conf.cpp +++ b/src/core/conf.cpp @@ -14,6 +14,11 @@ namespace mob::details { using key_value_map = std::map>; using section_map = std::map>; + static std::unordered_map s_configuration_values{ + {mob::config::release, "Release"}, + {mob::config::debug, "Debug"}, + {mob::config::relwithdebinfo, "RelWithDebInfo"}}; + // holds all the options not related to tasks (global, tools, paths, etc.) static section_map g_conf; @@ -27,6 +32,18 @@ namespace mob::details { static int g_file_log_level = 5; static bool g_dry = false; + // check if the two given string are equals case-insensitive + // + bool case_insensitive_equals(std::string_view lhs, std::string_view rhs) + { + // _strcmpi does not have a n-overload, and since string_view is not + // necessarily null-terminated, _strmcpi cannot be safely used + return std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs), + std::end(rhs), [](auto&& c1, auto&& c2) { + return ::tolower(c1) == ::tolower(c2); + }); + } + bool bool_from_string(std::string_view s) { return (s == "true" || s == "yes" || s == "1"); @@ -85,6 +102,17 @@ namespace mob::details { kitor->second = value; } + config string_to_config(std::string_view value) + { + for (const auto& [c, v] : s_configuration_values) { + if (case_insensitive_equals(value, v)) { + return c; + } + } + + gcx().bail_out(context::conf, "invalid configuration '{}'", value); + } + // sets the given option, adds it if it doesn't exist; used when setting options // from the master ini // @@ -184,6 +212,28 @@ namespace mob::details { g_tasks[task_name][key] = std::move(value); } + // read a CMake constant from the configuration + // + template + T parse_cmake_value(std::string_view section, std::string_view key, + std::string_view value, + std::unordered_map const& values) + { + for (const auto& [value_c, value_s] : values) { + if (case_insensitive_equals(value_s, value)) { + return value_c; + } + } + + // build a string containing allowed value for logging + std::vector values_s; + for (const auto& [value_c, value_s] : values) { + values_s.push_back(value_s); + } + gcx().bail_out(context::conf, "bad value '{}' for {}/{} (expected one of {})", + value, section, key, join(values_s, ", ", std::string{})); + } + } // namespace mob::details namespace mob { @@ -644,6 +694,11 @@ namespace mob { return {}; } + conf_build_types conf::build_types() + { + return {}; + } + conf_paths conf::path() { return {}; @@ -666,57 +721,52 @@ namespace mob { return details::g_dry; } - conf_task::conf_task(std::vector names) : names_(std::move(names)) {} + // use appropriate case for the below constants since we will be using them in + // to_string, although most of cmake and msbuild is case-insensitive so it will + // not matter much in the end - std::string conf_task::get(std::string_view key) const + static std::unordered_map constant_values{ + {conf_cmake::always, "ALWAYS"}, + {conf_cmake::lazy, "LAZY"}, + {conf_cmake::never, "NEVER"}}; + + std::string conf_cmake::to_string(constant c) { - return details::get_string_for_task(names_, key); + return std::string{constant_values.at(c)}; } - bool conf_task::get_bool(std::string_view key) const + conf_cmake::conf_cmake() : conf_section("cmake") {} + + conf_cmake::constant conf_cmake::install_message() const { - return details::get_bool_for_task(names_, key); + return details::parse_cmake_value( + name(), "install_message", details::get_string(name(), "install_message"), + constant_values); } - bool conf_cmake::cmake_constant::is_equivalent(std::string_view other) const + std::string conf_cmake::host() const { - // _strcmpi does not have a n-overload, and since string_view is not - // necessarily null-terminated, _strmcpi cannot be safely used - return std::equal(std::begin(value_), std::end(value_), std::begin(other), - std::end(other), [](auto&& c1, auto&& c2) { - return ::tolower(c1) == ::tolower(c2); - }); + return details::get_string(name(), "host"); } - conf_cmake::conf_cmake() : conf_section("cmake") {} - - const conf_cmake::cmake_constant conf_cmake::ALWAYS{"always"}; - const conf_cmake::cmake_constant conf_cmake::LAZY{"lazy"}; - const conf_cmake::cmake_constant conf_cmake::NEVER{"never"}; + conf_task::conf_task(std::vector names) : names_(std::move(names)) {} - conf_cmake::cmake_constant conf_cmake::install_message() const + std::string conf_task::get(std::string_view key) const { - return read_cmake_constant("install_message", {ALWAYS, LAZY, NEVER}); + return details::get_string_for_task(names_, key); } - std::string conf_cmake::host() const + bool conf_task::get_bool(std::string_view key) const { - return details::get_string(name(), "host"); + return details::get_bool_for_task(names_, key); } - conf_cmake::cmake_constant - conf_cmake::read_cmake_constant(std::string_view key, - std::vector const& allowed) const + mob::config conf_task::configuration() const { - const auto value = details::get_string(name(), key); - for (const auto& constant : allowed) { - if (constant.is_equivalent(value)) { - return constant; - } - } - - gcx().bail_out(context::conf, "bad value '{}' for {}/{} (expected one of {})", - value, name(), key, join(allowed, ", ", std::string{})); + return details::parse_cmake_value( + names_[0], "configuration", + details::get_string_for_task(names_, "configuration"), + details::s_configuration_values); } conf_tools::conf_tools() : conf_section("tools") {} @@ -725,6 +775,8 @@ namespace mob { conf_versions::conf_versions() : conf_section("versions") {} + conf_build_types::conf_build_types() : conf_section("build-types") {} + conf_prebuilt::conf_prebuilt() : conf_section("prebuilt") {} conf_paths::conf_paths() : conf_section("paths") {} diff --git a/src/core/conf.h b/src/core/conf.h index 891fc83..2589394 100644 --- a/src/core/conf.h +++ b/src/core/conf.h @@ -1,5 +1,7 @@ #pragma once +#include "../utility.h" + // these shouldn't be called directly, they're used by some of the template // below // @@ -9,6 +11,18 @@ namespace mob::details { // std::string get_string(std::string_view section, std::string_view key); + // convert a string to the given type + template + T string_to(std::string_view value); + + config string_to_config(std::string_view value); + + template <> + inline config string_to(std::string_view value) + { + return string_to_config(value); + } + // calls get_string(), converts to bool // bool get_bool(std::string_view section, std::string_view key); @@ -48,7 +62,14 @@ namespace mob { public: DefaultType get(std::string_view key) const { - return details::get_string(name_, key); + const auto value = details::get_string(name_, key); + + if constexpr (std::is_convertible_v) { + return value; + } + else { + return details::string_to(value); + } } // undefined @@ -102,6 +123,29 @@ namespace mob { bool build() const { return get("build_task"); } }; + // options in [cmake] + // + class conf_cmake : conf_section { + public: + enum class constant { always, lazy, never }; + using enum constant; + + static std::string to_string(constant m); + + conf_cmake(); + + // specify the value for CMAKE_INSTALL_MESSAGE + // + constant install_message() const; + + // specify the toolset host configuration, if any, this is equivalent + // to -T host=XXX on the command line + // + // an empty string means no host configured + // + std::string host() const; + }; + // options in [task] or [task_name:task] // class conf_task { @@ -141,64 +185,16 @@ namespace mob { return get("remote_push_default_origin"); } + // specify the configuration to build + // + mob::config configuration() const; + private: std::vector names_; bool get_bool(std::string_view name) const; }; - // options in [task] or [task_name:task] - // - class conf_cmake : conf_section { - public: - class cmake_constant { - std::string value_; - - // check if the given string is equivalent to this constant - // - bool is_equivalent(std::string_view other) const; - - friend class conf_cmake; - - public: - constexpr cmake_constant(std::string_view value) : value_{value} {} - constexpr const auto& value() const { return value_; } - constexpr operator const std::string&() const { return value(); } - - friend bool operator==(cmake_constant const& lhs, cmake_constant const& rhs) - { - return lhs.is_equivalent(rhs.value()); - } - friend bool operator!=(cmake_constant const& lhs, cmake_constant const& rhs) - { - return !(lhs == rhs); - } - }; - - static const cmake_constant ALWAYS; - static const cmake_constant LAZY; - static const cmake_constant NEVER; - - public: - conf_cmake(); - - // specify the value for CMAKE_INSTALL_MESSAGE - // - cmake_constant install_message() const; - - // specify the toolset host configuration, if any, this is equivalent - // to -T host=XXX on the command line - // - // an empty string means no host configured - // - std::string host() const; - - private: - cmake_constant - read_cmake_constant(std::string_view key, - std::vector const& allowed) const; - }; - // options in [tools] // class conf_tools : public conf_section { @@ -220,6 +216,13 @@ namespace mob { conf_versions(); }; + // options in [build-types] + // + class conf_build_types : public conf_section { + public: + conf_build_types(); + }; + // options in [prebuilt] // class conf_prebuilt : public conf_section { @@ -284,6 +287,7 @@ namespace mob { conf_transifex transifex(); conf_prebuilt prebuilt(); conf_versions version(); + conf_build_types build_types(); conf_paths path(); // opens the log file, creates the directory if needed diff --git a/src/tasks/boost.cpp b/src/tasks/boost.cpp index b5108e7..f64038e 100644 --- a/src/tasks/boost.cpp +++ b/src/tasks/boost.cpp @@ -218,23 +218,28 @@ namespace mob::tasks { bootstrap(); } - // some libraries only need some variants, avoid building the unnecessary - // ones + // we do not need all variants of all components but since people should + // usually be using the pre-built, we can build everything without losing too + // much time and it is much easier to deal when mixing + // + // note: filesystem is only required by USVFS I think, so maybe think about + // removing it if we switch to std::filesystem in USVFS + // + clipp::arg_list components{"thread", "date_time", "filesystem", "locale", + "program_options"}; // static link, static runtime, x64 - do_b2({"thread", "date_time", "filesystem", "locale", "program_options"}, - "static", "static", arch::x64); + do_b2(components, "static", "static", arch::x64); - // static link, static runtime, x86, required by usvfs 32-bit - do_b2({"thread", "date_time", "filesystem", "locale"}, "static", "static", - arch::x86); + // static link, static/shared runtime, x86, required by usvfs 32-bit + do_b2(components, "static", "static", arch::x86); + do_b2(components, "static", "shared", arch::x86); // static link, shared runtime, x64 - do_b2({"thread", "date_time", "locale", "program_options"}, "static", "shared", - arch::x64); + do_b2(components, "static", "shared", arch::x64); // shared link, shared runtime, x64 - do_b2({"thread", "date_time", "atomic"}, "shared", "shared", arch::x64); + do_b2(components, "shared", "shared", arch::x64); } void boost::do_b2(const std::vector& components, diff --git a/src/tasks/gtest.cpp b/src/tasks/gtest.cpp index c262ad9..359a433 100644 --- a/src/tasks/gtest.cpp +++ b/src/tasks/gtest.cpp @@ -5,25 +5,24 @@ namespace mob::tasks { namespace { - cmake create_cmake_tool(arch a, cmake::ops o = cmake::generate) + cmake create_cmake_tool(arch a, config config, cmake::ops o = cmake::generate) { - const std::string build_dir = (a == arch::x64 ? "build" : "build_32"); - return std::move(cmake(o) .generator(cmake::vs) .architecture(a) .arg("-Wno-deprecated") .arg("-Dgtest_force_shared_crt=ON") - .prefix(gtest::source_path() / build_dir) + .prefix(gtest::build_path(a, config)) .root(gtest::source_path())); } - msbuild create_msbuild_tool(arch a, msbuild::ops o = msbuild::build) + msbuild create_msbuild_tool(arch a, config config, + msbuild::ops o = msbuild::build) { - const fs::path build_path = create_cmake_tool(a).build_path(); + const fs::path build_path = create_cmake_tool(a, config).build_path(); - return std::move( - msbuild(o).architecture(a).solution(build_path / "INSTALL.vcxproj")); + return std::move(msbuild(o).architecture(a).configuration(config).solution( + build_path / "INSTALL.vcxproj")); } } // namespace @@ -45,6 +44,12 @@ namespace mob::tasks { return conf().path().build() / "googletest"; } + fs::path gtest::build_path(arch a, config c) + { + return source_path() / "build" / (a == arch::x64 ? "x64" : "Win32") / + msbuild::configuration_name(c); + } + void gtest::do_clean(clean c) { if (is_set(c, clean::reclone)) { @@ -53,13 +58,15 @@ namespace mob::tasks { } if (is_set(c, clean::reconfigure)) { - run_tool(create_cmake_tool(arch::x86, cmake::clean)); - run_tool(create_cmake_tool(arch::x64, cmake::clean)); + run_tool(create_cmake_tool(arch::x86, config::release, cmake::clean)); + run_tool(create_cmake_tool(arch::x64, config::release, cmake::clean)); } if (is_set(c, clean::rebuild)) { - run_tool(create_msbuild_tool(arch::x86, msbuild::clean)); - run_tool(create_msbuild_tool(arch::x64, msbuild::clean)); + run_tool(create_msbuild_tool(arch::x86, config::release, msbuild::clean)); + run_tool(create_msbuild_tool(arch::x86, config::debug, msbuild::clean)); + run_tool(create_msbuild_tool(arch::x64, config::release, msbuild::clean)); + run_tool(create_msbuild_tool(arch::x64, config::debug, msbuild::clean)); } } @@ -73,15 +80,24 @@ namespace mob::tasks { void gtest::do_build_and_install() { + op::create_directories(cx(), gtest::build_path(arch::x64).parent_path()); + op::create_directories(cx(), gtest::build_path(arch::x86).parent_path()); + parallel({{"gtest64", [&] { - run_tool(create_cmake_tool(arch::x64)); - run_tool(create_msbuild_tool(arch::x64)); + run_tool(create_cmake_tool(arch::x64, config::release)); + run_tool(create_msbuild_tool(arch::x64, config::release)); + + run_tool(create_cmake_tool(arch::x64, config::debug)); + run_tool(create_msbuild_tool(arch::x64, config::debug)); }}, {"gtest32", [&] { - run_tool(create_cmake_tool(arch::x86)); - run_tool(create_msbuild_tool(arch::x86)); + run_tool(create_cmake_tool(arch::x86, config::release)); + run_tool(create_msbuild_tool(arch::x86, config::release)); + + run_tool(create_cmake_tool(arch::x86, config::debug)); + run_tool(create_msbuild_tool(arch::x86, config::debug)); }}}); } diff --git a/src/tasks/libbsarch.cpp b/src/tasks/libbsarch.cpp index 0b8d87b..de41c56 100644 --- a/src/tasks/libbsarch.cpp +++ b/src/tasks/libbsarch.cpp @@ -7,7 +7,9 @@ namespace mob::tasks { std::string dir_name() { - return "libbsarch-" + libbsarch::version() + "-release-x64"; + return "libbsarch-" + libbsarch::version() + "-" + + (libbsarch::build_type() == config::debug ? "debug" : "release") + + "-x64"; } url source_url() @@ -25,6 +27,11 @@ namespace mob::tasks { return conf().version().get("libbsarch"); } + config libbsarch::build_type() + { + return conf().build_types().get("libbsarch"); + } + bool libbsarch::prebuilt() { return false; diff --git a/src/tasks/modorganizer.cpp b/src/tasks/modorganizer.cpp index af288c3..2678c87 100644 --- a/src/tasks/modorganizer.cpp +++ b/src/tasks/modorganizer.cpp @@ -200,36 +200,40 @@ namespace mob::tasks { cmake modorganizer::create_cmake_tool(cmake::ops o) { - return create_cmake_tool(source_path(), o); - } - - cmake modorganizer::create_cmake_tool(const fs::path& root, cmake::ops o) - { - return std::move(cmake(o) - .generator(cmake::vs) - .def("CMAKE_INSTALL_PREFIX:PATH", conf().path().install()) - .def("DEPENDENCIES_DIR", conf().path().build()) - .def("BOOST_ROOT", boost::source_path()) - .def("BOOST_LIBRARYDIR", boost::lib_path(arch::x64)) - .def("SPDLOG_ROOT", spdlog::source_path()) - .def("LOOT_PATH", libloot::source_path()) - .def("LZ4_ROOT", lz4::source_path()) - .def("QT_ROOT", qt::installation_path()) - .def("ZLIB_ROOT", zlib::source_path()) - .def("PYTHON_ROOT", python::source_path()) - .def("SEVENZ_ROOT", sevenz::source_path()) - .def("LIBBSARCH_ROOT", libbsarch::source_path()) - .def("BOOST_DI_ROOT", boost_di::source_path()) - .def("GTEST_ROOT", gtest::source_path()) - .def("OPENSSL_ROOT_DIR", openssl::source_path()) - .root(root)); + return create_cmake_tool(source_path(), o, task_conf().configuration()); + } + + cmake modorganizer::create_cmake_tool(const fs::path& root, cmake::ops o, config c) + { + return std::move( + cmake(o) + .generator(cmake::vs) + .def("CMAKE_INSTALL_PREFIX:PATH", conf().path().install()) + .def("DEPENDENCIES_DIR", conf().path().build()) + .def("BOOST_ROOT", boost::source_path()) + .def("BOOST_LIBRARYDIR", boost::lib_path(arch::x64)) + .def("SPDLOG_ROOT", spdlog::source_path()) + .def("LOOT_PATH", libloot::source_path()) + .def("LZ4_ROOT", lz4::source_path()) + .def("QT_ROOT", qt::installation_path()) + .def("ZLIB_ROOT", zlib::source_path()) + .def("PYTHON_ROOT", python::source_path()) + .def("SEVENZ_ROOT", sevenz::source_path()) + .def("LIBBSARCH_ROOT", libbsarch::source_path()) + .def("BOOST_DI_ROOT", boost_di::source_path()) + // gtest has no RelWithDebInfo, so simply use Debug/Release + .def("GTEST_ROOT", + gtest::build_path(arch::x64, c == config::debug ? config::debug + : config::release)) + .def("OPENSSL_ROOT_DIR", openssl::source_path()) + .root(root)); } msbuild modorganizer::create_msbuild_tool(msbuild::ops o) { return std::move(msbuild(o) .solution(project_file_path()) - .config("RelWithDebInfo") + .configuration(task_conf().configuration()) .architecture(arch::x64)); } diff --git a/src/tasks/pyqt.cpp b/src/tasks/pyqt.cpp index 20e8397..d27d592 100644 --- a/src/tasks/pyqt.cpp +++ b/src/tasks/pyqt.cpp @@ -47,7 +47,9 @@ namespace mob::tasks { url prebuilt_url() { - return make_prebuilt_url("PyQt6_gpl-prebuilt-" + pyqt::version() + ".7z"); + return make_prebuilt_url( + "PyQt6_gpl-prebuilt-" + pyqt::version() + + (pyqt::build_type() == config::debug ? "-debug" : "") + ".7z"); } // file created by sip-module.exe @@ -86,6 +88,11 @@ namespace mob::tasks { return source_path() / "build"; } + config pyqt::build_type() + { + return conf().build_types().get("pyqt"); + } + std::string pyqt::pyqt_sip_module_name() { return "PyQt6.sip"; @@ -166,7 +173,24 @@ namespace mob::tasks { void pyqt::build_and_install_from_source() { // use pip to install the pyqt builder - run_tool(pip(pip::install).package("PyQt-builder").version(builder_version())); + if (python::build_type() == config::debug) { + // PyQt-builder has sip as a dependency, so installing it directly will + // replace the sip we have installed manually, but the installed sip will + // not work (see comment in sip::build() for details) + // + // the workaround is to install the dependencies manually (only packaging), + // and then use a --no-dependencies install with pip + // + run_tool(pip(pip::install).package("packaging")); + run_tool(pip(pip::install) + .package("PyQt-builder") + .no_dependencies() + .version(builder_version())); + } + else { + run_tool( + pip(pip::install).package("PyQt-builder").version(builder_version())); + } // patch for builder.py run_tool(patcher() @@ -249,9 +273,30 @@ namespace mob::tasks { cx().trace(context::bypass, "pyqt already installed"); } else { - // run `pip install` on the generated PyQt6_sip-XX.tar.gz file - run_tool( - pip(pip::install).file(conf().path().cache() / sip_install_file())); + if (python::build_type() == config::debug) { + + const auto pyqt_sip_folder = source_path() / "PyQt6_sip" / + ("pyqt6_sip-" + sip::version_for_pyqt()); + + // in debug, we need to do stuff manually because pip does not work + // as expected + run_tool(extractor() + .file(conf().path().cache() / sip_install_file()) + .output(pyqt_sip_folder.parent_path())); + + run_tool(mob::python() + .root(pyqt_sip_folder) + .arg("setup.py") + .arg("build") + .arg("--debug")); + + run_tool(pip(pip::install).file(pyqt_sip_folder)); + } + else { + // run `pip install` on the generated PyQt6_sip-XX.tar.gz file + run_tool( + pip(pip::install).file(conf().path().cache() / sip_install_file())); + } // done, create the bypass file installed_bypass.create(); @@ -267,13 +312,17 @@ namespace mob::tasks { // python-XX/PCBuild/amd64, those are needed by PyQt6 when building several // projects - op::copy_file_to_dir_if_better(cx(), qt::bin_path() / "Qt6Core.dll", - python::build_path(), - op::unsafe); // source file is outside prefix + const std::vector dlls{"Qt6Core", "Qt6Xml"}; - op::copy_file_to_dir_if_better(cx(), qt::bin_path() / "Qt6Xml.dll", - python::build_path(), - op::unsafe); // source file is outside prefix + for (auto dll : dlls) { + if (build_type() == config::debug) { + dll += "d"; + } + dll += ".dll"; + op::copy_file_to_dir_if_better( + cx(), qt::bin_path() / dll, python::build_path(), + op::unsafe); // source file is outside prefix + } // installation of PyQt6 python files (.pyd, etc.) is done // by the python plugin directly diff --git a/src/tasks/python.cpp b/src/tasks/python.cpp index 2748acc..b70d804 100644 --- a/src/tasks/python.cpp +++ b/src/tasks/python.cpp @@ -30,7 +30,9 @@ namespace mob::tasks { url prebuilt_url() { - return make_prebuilt_url("python-prebuilt-" + version_without_v() + ".7z"); + return make_prebuilt_url( + "python-prebuilt-" + version_without_v() + + (python::build_type() == config::debug ? "-debug" : "") + ".7z"); } fs::path solution_file() @@ -89,6 +91,11 @@ namespace mob::tasks { return source_path() / "PCBuild" / "amd64"; } + config python::build_type() + { + return conf().build_types().get("python"); + } + void python::do_clean(clean c) { if (prebuilt()) { @@ -190,16 +197,22 @@ namespace mob::tasks { const auto bat = source_path() / "python.bat"; + auto p = process() + .binary(bat) + .arg(fs::path("PC/layout")) + .arg("--source", source_path()) + .arg("--build", build_path()) + .arg("--temp", (build_path() / "pythoncore_temp")) + .arg("--copy", (build_path() / "pythoncore")) + .arg("--preset-embed") + .cwd(source_path()); + + if (build_type() == config::debug) { + p = p.arg("--debug"); + } + // package libs into pythonXX.zip - run_tool(process_runner(process() - .binary(bat) - .arg(fs::path("PC/layout")) - .arg("--source", source_path()) - .arg("--build", build_path()) - .arg("--temp", (build_path() / "pythoncore_temp")) - .arg("--copy", (build_path() / "pythoncore")) - .arg("--preset-embed") - .cwd(source_path()))); + run_tool(process_runner(p)); } void python::copy_files() @@ -210,7 +223,9 @@ namespace mob::tasks { // pdbs op::copy_file_to_dir_if_better( - cx(), build_path() / ("python" + version_for_dll() + ".pdb"), + cx(), + build_path() / ("python" + version_for_dll() + + (build_type() == config::debug ? "_d" : "") + ".pdb"), conf().path().install_pdbs()); // dlls and python libraries are installed by the python plugin @@ -229,6 +244,7 @@ namespace mob::tasks { .solution(solution_file()) .targets({"python", "pythonw", "python3dll", "select", "pyexpat", "unicodedata", "_queue", "_bz2", "_ssl", "_overlapped"}) + .configuration(build_type()) .properties( {"bz2Dir=" + path_to_utf8(bzip2::source_path()), "zlibDir=" + path_to_utf8(zlib::source_path()), @@ -238,7 +254,8 @@ namespace mob::tasks { fs::path python::python_exe() { - return build_path() / "python.exe"; + return build_path() / + (build_type() == config::debug ? "python_d.exe" : "python.exe"); } fs::path python::include_path() diff --git a/src/tasks/sip.cpp b/src/tasks/sip.cpp index 36b8cdb..aed6e34 100644 --- a/src/tasks/sip.cpp +++ b/src/tasks/sip.cpp @@ -140,7 +140,16 @@ namespace mob::tasks { void sip::build() { - run_tool(pip(pip::install).file(source_path())); + if (python::build_type() == config::debug) { + // if Python is build in debug mode, fall back to old setup.py because pip + // install seems to generated broken script wrapper that point to a + // non-existing python.exe instead of python_d.exe + run_tool(pip(pip::install).package("setuptools")); + run_tool(mob::python().root(source_path()).arg("setup.py").arg("install")); + } + else { + run_tool(pip(pip::install).file(source_path())); + } } void sip::convert_script_file_to_acp(const std::string& filename) diff --git a/src/tasks/tasks.h b/src/tasks/tasks.h index a5dbaf8..25efa26 100644 --- a/src/tasks/tasks.h +++ b/src/tasks/tasks.h @@ -97,6 +97,7 @@ namespace mob::tasks { static std::string version(); static bool prebuilt(); static fs::path source_path(); + static fs::path build_path(arch = arch::x64, config = config::release); protected: void do_clean(clean c) override; @@ -123,6 +124,7 @@ namespace mob::tasks { libbsarch(); static std::string version(); + static config build_type(); static bool prebuilt(); static fs::path source_path(); @@ -197,7 +199,8 @@ namespace mob::tasks { // projects, used internally, but also by the cmake command // static cmake create_cmake_tool(const fs::path& root, - cmake::ops o = cmake::generate); + cmake::ops o = cmake::generate, + config config = config::relwithdebinfo); // flags for some MO projects enum flags { @@ -360,6 +363,7 @@ namespace mob::tasks { static fs::path source_path(); static fs::path build_path(); + static config build_type(); // "PyQt5.sip", used both in pyqt and sip // @@ -419,6 +423,10 @@ namespace mob::tasks { // static fs::path site_packages_path(); + // configuration to build + // + static config build_type(); + protected: void do_clean(clean c) override; void do_fetch() override; @@ -649,7 +657,8 @@ namespace mob::tasks { void fetch_from_source(); void build_and_install_from_source(); - msbuild create_msbuild_tool(arch a, msbuild::ops o = msbuild::build) const; + msbuild create_msbuild_tool(arch, msbuild::ops = msbuild::build, + config = config::release) const; }; class zlib : public basic_task { diff --git a/src/tasks/usvfs.cpp b/src/tasks/usvfs.cpp index 0263cf4..a61e277 100644 --- a/src/tasks/usvfs.cpp +++ b/src/tasks/usvfs.cpp @@ -42,8 +42,10 @@ namespace mob::tasks { if (is_set(c, clean::rebuild)) { // msbuild clean - run_tool(create_msbuild_tool(arch::x86, msbuild::clean)); - run_tool(create_msbuild_tool(arch::x64, msbuild::clean)); + run_tool(create_msbuild_tool(arch::x86, msbuild::clean, + task_conf().configuration())); + run_tool(create_msbuild_tool(arch::x64, msbuild::clean, + task_conf().configuration())); } } @@ -71,7 +73,7 @@ namespace mob::tasks { run_tool(create_msbuild_tool(arch::x64)); } - msbuild usvfs::create_msbuild_tool(arch a, msbuild::ops o) const + msbuild usvfs::create_msbuild_tool(arch a, msbuild::ops o, config c) const { // usvfs doesn't use "Win32" for 32-bit, it uses "x86" // @@ -90,6 +92,7 @@ namespace mob::tasks { return std::move( msbuild(o) .platform(plat) + .configuration(c) .targets({"usvfs_proxy"}) .solution(source_path() / "vsbuild" / "usvfs.sln") .env(env::vs(a) diff --git a/src/tools/cmake.cpp b/src/tools/cmake.cpp index 1bf83a6..0f277e0 100644 --- a/src/tools/cmake.cpp +++ b/src/tools/cmake.cpp @@ -129,7 +129,7 @@ namespace mob { .binary(binary()) .arg("-DCMAKE_BUILD_TYPE=Release") .arg("-DCMAKE_INSTALL_MESSAGE=" + - conf().cmake().install_message().value()) + conf_cmake::to_string(conf().cmake().install_message())) .arg("--log-level=ERROR") .arg("--no-warn-unused-cli"); diff --git a/src/tools/cmake.h b/src/tools/cmake.h index 41a50dc..ea8a3a6 100644 --- a/src/tools/cmake.h +++ b/src/tools/cmake.h @@ -14,23 +14,25 @@ namespace mob { // type of build files generated // - enum generators { + enum class generators { // generates build files for visual studio vs = 0x01, // generates build files for jom/nmake jom = 0x02 }; + using enum generators; // what run() will do // - enum ops { + enum class ops { // generates the build files generate = 1, // cleans the build files so they're regenerated from scratch clean }; + using enum ops; cmake(ops o = generate); diff --git a/src/tools/msbuild.cpp b/src/tools/msbuild.cpp index c7accd1..8eb8b7f 100644 --- a/src/tools/msbuild.cpp +++ b/src/tools/msbuild.cpp @@ -7,8 +7,8 @@ namespace mob { msbuild::msbuild(ops o) - : basic_process_runner("msbuild"), op_(o), config_("Release"), arch_(arch::def), - flags_(noflags) + : basic_process_runner("msbuild"), op_(o), config_(config::release), + arch_(arch::def), flags_(noflags) { } @@ -17,6 +17,20 @@ namespace mob { return conf().tool().get("msbuild"); } + std::string msbuild::configuration_name(config c) + { + switch (c) { + case config::debug: + return "Debug"; + case config::release: + return "Release"; + case config::relwithdebinfo: + [[fallthrough]]; + default: + return "RelWithDebInfo"; + } + } + msbuild& msbuild::solution(const fs::path& sln) { sln_ = sln; @@ -35,9 +49,9 @@ namespace mob { return *this; } - msbuild& msbuild::config(const std::string& s) + msbuild& msbuild::configuration(config c) { - config_ = s; + config_ = c; return *this; } @@ -151,7 +165,7 @@ namespace mob { .arg("-property:EnforceProcessCountAcrossBuilds=true"); } - p.arg("-property:Configuration=", config_, process::quote) + p.arg("-property:Configuration=", configuration_name(config_), process::quote) .arg("-property:PlatformToolset=" + toolset) .arg("-property:WindowsTargetPlatformVersion=" + vs::sdk()) .arg("-property:Platform=", platform_property(), process::quote) diff --git a/src/tools/msbuild.h b/src/tools/msbuild.h index e70aeff..7cde18f 100644 --- a/src/tools/msbuild.h +++ b/src/tools/msbuild.h @@ -1,6 +1,8 @@ #pragma once +#include "../core/conf.h" #include "../core/env.h" +#include "cmake.h" namespace mob { @@ -14,11 +16,17 @@ namespace mob { // static fs::path binary(); - enum flags_t { noflags = 0x00, single_job = 0x01, allow_failure = 0x02 }; + // retrieve the name of the configuration for the given config + // + static std::string configuration_name(config c); + + enum class flags_t { noflags = 0x00, single_job = 0x01, allow_failure = 0x02 }; + using enum flags_t; // what run() should do // - enum ops { build = 1, clean }; + enum class ops { build = 1, clean }; + using enum ops; msbuild(ops o = build); @@ -36,7 +44,7 @@ namespace mob { // sets "-property:Configuration=s" // - msbuild& config(const std::string& s); + msbuild& configuration(config config); // sets "-property:Platform=s"; if not set, uses architecture() to figure // it out @@ -69,7 +77,7 @@ namespace mob { fs::path sln_; std::vector targets_; std::vector props_; - std::string config_; + config config_; std::string platform_; arch arch_; flags_t flags_; diff --git a/src/tools/python.cpp b/src/tools/python.cpp index 255395f..46c79bc 100644 --- a/src/tools/python.cpp +++ b/src/tools/python.cpp @@ -49,7 +49,7 @@ namespace mob { execute_and_join(p); } - pip::pip(ops op) : basic_process_runner("pip"), op_(op) {} + pip::pip(ops op) : basic_process_runner("pip"), op_(op), no_deps_{false} {} pip& pip::package(const std::string& s) { @@ -69,6 +69,12 @@ namespace mob { return *this; } + pip& pip::no_dependencies() + { + no_deps_ = true; + return *this; + } + void pip::do_run() { switch (op_) { @@ -141,11 +147,21 @@ namespace mob { .arg("--no-warn-script-location") .arg("--disable-pip-version-check"); - if (!package_.empty()) - p.arg(package_ + "==" + version_); + if (!package_.empty()) { + if (version_.empty()) { + p.arg(package_); + } + else { + p.arg(package_ + "==" + version_); + } + } else if (!file_.empty()) p.arg(file_); + if (no_deps_) { + p.arg("--no-dependencies"); + } + p.env(this_env::get().set("PYTHONUTF8", "1")); execute_and_join(p); diff --git a/src/tools/tools.h b/src/tools/tools.h index 158cae1..3d16fef 100644 --- a/src/tools/tools.h +++ b/src/tools/tools.h @@ -703,6 +703,10 @@ namespace mob { pip& version(const std::string& s); pip& file(const fs::path& p); + // do not install dependencies for the package + // + pip& no_dependencies(); + protected: // runs pip // @@ -717,6 +721,9 @@ namespace mob { std::string version_; fs::path file_; + // no depndencies + bool no_deps_; + // runs `-m ensurepip`, then upgrades pip // void do_ensure(); diff --git a/src/utility.h b/src/utility.h index 1a73a29..a0becf4 100644 --- a/src/utility.h +++ b/src/utility.h @@ -46,6 +46,8 @@ namespace mob { def = x64 }; + enum class config { debug, relwithdebinfo, release }; + class url; // returns a url for a prebuilt binary having the given filename; prebuilts are