From f3493a85b2d82aebab44a5580443413040ae5794 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 3 Oct 2025 22:21:42 +0000 Subject: [PATCH 01/13] Modularize --- taskfiles/deps/cmake-libs.yaml | 359 +++++++++++++++++++ taskfiles/deps/core.yaml | 46 +++ taskfiles/deps/log-viewer.yaml | 14 + taskfiles/deps/main.yaml | 622 ++------------------------------- taskfiles/deps/metadata.yaml | 28 ++ taskfiles/deps/misc-libs.yaml | 136 +++++++ taskfiles/deps/spider.yaml | 55 +++ taskfiles/deps/utils.yaml | 10 - 8 files changed, 658 insertions(+), 612 deletions(-) create mode 100644 taskfiles/deps/cmake-libs.yaml create mode 100644 taskfiles/deps/core.yaml create mode 100644 taskfiles/deps/log-viewer.yaml create mode 100644 taskfiles/deps/metadata.yaml create mode 100644 taskfiles/deps/misc-libs.yaml create mode 100644 taskfiles/deps/spider.yaml diff --git a/taskfiles/deps/cmake-libs.yaml b/taskfiles/deps/cmake-libs.yaml new file mode 100644 index 0000000000..5805991435 --- /dev/null +++ b/taskfiles/deps/cmake-libs.yaml @@ -0,0 +1,359 @@ +version: "3" + +# NOTE: For dependencies built using CMake, we set CMP0074 to NEW whenever: +# - a component's minimum required CMake version is less than 3.27 (where CMP0074 defaults to OLD); +# and +# - the component depends on another via `_ROOT`. +tasks: + absl: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_CXX_STANDARD=20" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "absl" + TARBALL_SHA256: "7262daa7c1711406248c10f41026d685e88223bc92817d16fb93c19adb57f669" + TARBALL_URL: "https://github.com/abseil/abseil-cpp/releases/download/20250512.0/\ + abseil-cpp-20250512.0.tar.gz" + + catch2: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DBUILD_TESTING=OFF" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_CXX_STANDARD=20" + - "-DCMAKE_CXX_STANDARD_REQUIRED=ON" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "Catch2" + TARBALL_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" + TARBALL_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" + + date: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "date" + TARBALL_SHA256: "7a390f200f0ccd207e8cff6757e04817c1a0aec3e327b006b7eb451c57ee3538" + TARBALL_URL: "https://github.com/HowardHinnant/date/archive/refs/tags/v3.0.1.tar.gz" + + fmt: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DFMT_DOC=OFF" + - "-DFMT_TEST=OFF" + LIB_NAME: "{{.G_FMT_LIB_NAME}}" + TARBALL_SHA256: "bc23066d87ab3168f27cef3e97d545fa63314f5c79df5ea444d41d56f962c6af" + TARBALL_URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.2.0.tar.gz" + + liblzma: + internal: true + vars: + COMMON_CMAKE_GEN_ARGS: + - "-DBUILD_TESTING=OFF" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DXZ_DOC=OFF" + - "-DXZ_TOOL_LZMADEC=OFF" + - "-DXZ_TOOL_LZMAINFO=OFF" + - "-DXZ_TOOL_SCRIPTS=OFF" + - "-DXZ_TOOL_SYMLINKS_LZMA=OFF" + - "-DXZ_TOOL_XZ=OFF" + - "-DXZ_TOOL_XZDEC=OFF" + TARBALL_SHA256: "507825b599356c10dca1cd720c9d0d0c9d5400b9de300af00e4d1ea150795543" + TARBALL_URL: "https://github.com/tukaani-project/xz/releases/download/v5.8.1/xz-5.8.1.tar.gz" + run: "once" + deps: + - task: "liblzma-install" + vars: + BUILD_SHARED_LIBS: true + COMMON_CMAKE_GEN_ARGS: + ref: ".COMMON_CMAKE_GEN_ARGS" + TARBALL_SHA256: "{{.TARBALL_SHA256}}" + TARBALL_URL: "{{.TARBALL_URL}}" + - task: "liblzma-install" + vars: + BUILD_SHARED_LIBS: false + COMMON_CMAKE_GEN_ARGS: + ref: ".COMMON_CMAKE_GEN_ARGS" + TARBALL_SHA256: "{{.TARBALL_SHA256}}" + TARBALL_URL: "{{.TARBALL_URL}}" + + liblzma-install: + internal: true + requires: + vars: ["BUILD_SHARED_LIBS", "COMMON_CMAKE_GEN_ARGS", "TARBALL_SHA256", "TARBALL_URL"] + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DBUILD_SHARED_LIBS={{ if .BUILD_SHARED_LIBS }}ON{{ else }}OFF{{ end }}" + - >- + {{ join " " .COMMON_CMAKE_GEN_ARGS }} + LIB_NAME: "LibLZMA-{{ if .BUILD_SHARED_LIBS }}shared{{ else }}static{{ end }}" + TARBALL_SHA256: "{{.TARBALL_SHA256}}" + TARBALL_URL: "{{.TARBALL_URL}}" + + log-surgeon: + internal: true + run: "once" + deps: + - "fmt" + - "microsoft.gsl" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_GSL_LIB_NAME}}.cmake" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-Dlog_surgeon_BUILD_TESTING=OFF" + LIB_NAME: "log_surgeon" + TARBALL_SHA256: "69a99e0804a52c6b6397c5e7eabecc9bb4915d0145632c66fc63ad13678ff56a" + TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/a722d07.tar.gz" + + lz4: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DBUILD_SHARED_LIBS=ON" + - "-DBUILD_STATIC_LIBS=ON" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DLZ4_BUILD_CLI=OFF" + CMAKE_SOURCE_DIR: "build/cmake" + LIB_NAME: "lz4" + TARBALL_SHA256: "537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b" + TARBALL_URL: "https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz" + + mariadb-connector-cpp: + platforms: ["linux"] + preconditions: + # NOTE: The MariaDB connector is only required for building Spider, which is only supported on + # Ubuntu. `INSTALL_LAYOUT` is currently hardcoded to "DEB". To support other Linux distros: + # - Update the precondition accordingly. + # - Make `INSTALL_LAYOUT` configurable. + - >- + source /etc/os-release && [[ "$ID" == "ubuntu" ]] + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DUSE_SYSTEM_INSTALLED_LIB=ON" + - "-DINSTALL_LAYOUT=DEB" + LIB_NAME: "mariadb-connector-cpp" + TARBALL_URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/\ + tags/1.1.5.tar.gz" + TARBALL_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" + + microsoft.gsl: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DGSL_TEST=OFF" + LIB_NAME: "{{.G_GSL_LIB_NAME}}" + TARBALL_SHA256: "f0e32cb10654fea91ad56bde89170d78cfbf4363ee0b01d8f097de2ba49f6ce9" + TARBALL_URL: "https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.tar.gz" + + mongocxx: + internal: true + vars: + VERSION: "r4.1.1" + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DBUILD_SHARED_AND_STATIC_LIBS=ON" + - "-DBUILD_SHARED_LIBS_WITH_STATIC_MONGOC=ON" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DENABLE_UNINSTALL=OFF" + LIB_NAME: "mongocxx" + TARBALL_SHA256: "19dff3cf834a3e09229260f22a0325820a7e30c78b294db91794dd934776b33a" + TARBALL_URL: "https://github.com/mongodb/mongo-cxx-driver/releases/download/{{.VERSION}}\ + /mongo-cxx-driver-{{.VERSION}}.tar.gz" + + msgpack-cxx: + internal: true + vars: + VERSION: "7.0.0" + run: "once" + deps: + - task: "boost" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" + - "-DMSGPACK_BUILD_DOCS=OFF" + - "-DMSGPACK_CXX20=ON" + LIB_NAME: "msgpack-cxx" + TARBALL_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" + TARBALL_URL: "https://github.com/msgpack/msgpack-c/releases/download/cpp-{{.VERSION}}\ + /msgpack-cxx-{{.VERSION}}.tar.gz" + + nlohmann_json: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DJSON_BuildTests=OFF" + LIB_NAME: "nlohmann_json" + TARBALL_SHA256: "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406" + + # NOTE: We use the GitHub-generated source tarball for this version rather than the + # release tarball, since the latter is served from githubusercontent.com which is blocked + # by some developers' firewalls. The contents of the former are a superset of the latter. + TARBALL_URL: "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz" + + simdjson: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "simdjson" + TARBALL_SHA256: "07a1bb3587aac18fd6a10a83fe4ab09f1100ab39f0cb73baea1317826b9f9e0d" + TARBALL_URL: "https://github.com/simdjson/simdjson/archive/refs/tags/v3.13.0.tar.gz" + + spdlog: + internal: true + run: "once" + deps: + - "fmt" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" + - "-DSPDLOG_BUILD_EXAMPLE=OFF" + - "-DSPDLOG_BUILD_EXAMPLE_HO=OFF" + - "-DSPDLOG_FMT_EXTERNAL=ON" + LIB_NAME: "spdlog" + TARBALL_SHA256: "15a04e69c222eb6c01094b5c7ff8a249b36bb22788d72519646fb85feb267e67" + + # NOTE: Since spdlog depends on fmt, we need to choose a version of spdlog that's + # compatible with the version of fmt we use. + TARBALL_URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.3.tar.gz" + + utfcpp: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "utfcpp" + TARBALL_SHA256: "6920a6a5d6a04b9a89b2a89af7132f8acefd46e0c2a7b190350539e9213816c0" + TARBALL_URL: "https://github.com/nemtrif/utfcpp/archive/refs/tags/v4.0.6.tar.gz" + + yaml-cpp: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DBUILD_TESTING=OFF" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + LIB_NAME: "yaml-cpp" + TARBALL_SHA256: "43e6a9fcb146ad871515f0d0873947e5d497a1c9c60c58cb102a97b47208b7c3" + TARBALL_URL: "https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.tar.gz" + + ystdlib: + internal: true + run: "once" + deps: + - task: "boost" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-Dystdlib_BUILD_TESTING=OFF" + LIB_NAME: "ystdlib" + TARBALL_SHA256: "65990dc2bcc4a355c2181bfe31a7800f492309d1bcd340f52a34e85047e61bc8" + TARBALL_URL: "https://github.com/y-scope/ystdlib-cpp/archive/9ed78cd.tar.gz" + + zlib: + internal: true + vars: + LIB_NAME: "zlib" + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DBUILD_SHARED_LIBS=ON" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_INSTALL_PREFIX={{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" + - "-DZLIB_BUILD_EXAMPLES=OFF" + LIB_NAME: "{{.LIB_NAME}}" + TARBALL_SHA256: "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23" + TARBALL_URL: "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz" + + zstd: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DZSTD_BUILD_CONTRIB=OFF" + - "-DZSTD_BUILD_PROGRAMS=OFF" + - "-DZSTD_BUILD_SHARED=ON" + - "-DZSTD_BUILD_STATIC=ON" + - "-DZSTD_BUILD_TESTS=OFF" + CMAKE_SOURCE_DIR: "build/cmake" + LIB_NAME: "zstd" + TARBALL_SHA256: "eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" + TARBALL_URL: "https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz" diff --git a/taskfiles/deps/core.yaml b/taskfiles/deps/core.yaml new file mode 100644 index 0000000000..475024e21a --- /dev/null +++ b/taskfiles/deps/core.yaml @@ -0,0 +1,46 @@ +version: "3" + +tasks: + core: + deps: + - task: "init" + - task: "utils:clean-outdated-cpp-checksum-files" + cmds: + - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" + vars: + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-core.cmake" + DEP_TASK: "core-all-parallel" + - task: "utils:combine-cpp-checksum-files" + + core-all-parallel: + internal: true + vars: + CPP_CORE_TASKS: + - "absl" + - "antlr-jar" + - "antlr-runtime" + - "boost" + - "catch2" + - "date" + - "fmt" + - "liblzma" + - "log-surgeon" + - "lz4" + - "microsoft.gsl" + - "mongocxx" + - "msgpack-cxx" + - "nlohmann_json" + - "simdjson" + - "spdlog" + - "sqlite3" + - "utfcpp" + - "yaml-cpp" + - "ystdlib" + - "zlib" + - "zstd" + run: "once" + deps: + - for: + var: "CPP_CORE_TASKS" + task: "{{.ITEM}}" diff --git a/taskfiles/deps/log-viewer.yaml b/taskfiles/deps/log-viewer.yaml new file mode 100644 index 0000000000..10d623ef4e --- /dev/null +++ b/taskfiles/deps/log-viewer.yaml @@ -0,0 +1,14 @@ +version: "3" + +tasks: + log-viewer: + deps: + - task: "init" + cmds: + - task: "yscope-dev-utils:remote:download-and-extract-tar" + vars: + CHECKSUM_FILE: "{{.G_DEPS_LOG_VIEWER_CHECKSUM_FILE}}" + FILE_SHA256: "f9baaae0886439862204e8062c69dd49354cb14ad4e59f7c35779d544384955e" + OUTPUT_DIR: "{{.G_WEBUI_SRC_DIR}}/yscope-log-viewer" + TAR_FILE: "{{.G_BUILD_DIR}}/yscope-log-viewer.tar.gz" + URL: "https://github.com/y-scope/yscope-log-viewer/archive/3abe4cc.tar.gz" diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 9a6b0ab478..663295a63b 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -1,7 +1,22 @@ version: "3" includes: - utils: "./utils.yaml" + cmake-libs: + flatten: true + taskfile: "cmake-libs.yaml" + core: + flatten: true + taskfile: "core.yaml" + misc-libs: + flatten: true + taskfile: "misc-libs.yaml" + spider: + flatten: true + taskfile: "spider.yaml" + log-viewer: + flatten: true + taskfile: "log-viewer.yaml" + utils: "utils.yaml" yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" vars: @@ -26,614 +41,17 @@ vars: G_ANTLR_VERSION: "4.13.2" G_ANTLR_JAR_FILE: "{{.G_DEPS_CPP_DIR}}/antlr-{{.G_ANTLR_VERSION}}-complete.jar" -# NOTE: For dependencies built using CMake, we set CMP0074 to NEW whenever: -# - a component's minimum required CMake version is less than 3.27 (where CMP0074 defaults to OLD); -# and -# - the component depends on another via `_ROOT`. tasks: default: deps: - "core" - "log-viewer" - core: - deps: - - "utils:clean-outdated-cpp-checksum-files" - - "utils:init" - cmds: - - "rm -rf '{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}'" - - "mkdir -p '{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}'" - - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" - vars: - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" - CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-core.cmake" - DEP_TASK: "core-all-parallel" - - task: "utils:combine-cpp-checksum-files" - - log-viewer: - deps: - - "utils:init" - cmds: - - task: "yscope-dev-utils:remote:download-and-extract-tar" - vars: - CHECKSUM_FILE: "{{.G_DEPS_LOG_VIEWER_CHECKSUM_FILE}}" - FILE_SHA256: "f9baaae0886439862204e8062c69dd49354cb14ad4e59f7c35779d544384955e" - OUTPUT_DIR: "{{.G_WEBUI_SRC_DIR}}/yscope-log-viewer" - TAR_FILE: "{{.G_BUILD_DIR}}/yscope-log-viewer.tar.gz" - URL: "https://github.com/y-scope/yscope-log-viewer/archive/3abe4cc.tar.gz" - - core-all-parallel: + init: internal: true run: "once" - deps: - - task: "absl" - - task: "antlr-jar" - - task: "antlr-runtime" - - task: "boost" - - task: "catch2" - - task: "date" - - task: "fmt" - - task: "liblzma" - - task: "log-surgeon" - - task: "lz4" - - task: "microsoft.gsl" - - task: "mongocxx" - - task: "msgpack-cxx" - - task: "nlohmann_json" - - task: "simdjson" - - task: "spdlog" - - task: "sqlite3" - - task: "utfcpp" - - task: "yaml-cpp" - - task: "ystdlib" - - task: "zlib" - - task: "zstd" - - spider: - deps: - - "spider-dep" - platforms: ["linux"] - preconditions: - # NOTE: Spider is only supported on Ubuntu at present. - - >- - source /etc/os-release && [[ "$ID" == "ubuntu" ]] - cmds: - - task: "yscope-dev-utils:cmake:install-remote-tar" - vars: - CMAKE_PACKAGE_NAME: "spider" - TAR_SHA256: "43478b6da99aa2df50c960573ec68c6fcf67cdfbab39b5e314676218aa93b959" - TAR_URL: "https://github.com/y-scope/spider/archive/6a7bad3.tar.gz" - WORK_DIR: "{{.G_SPIDER_BUILD_DIR}}" - CMAKE_JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" - CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-spider.cmake" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DSPIDER_BUILD_TESTING=OFF" - - spider-dep: - internal: true - deps: - - "utils:clean-outdated-cpp-checksum-files" - - "utils:init" cmds: - - "rm -rf '{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}'" + - "mkdir -p '{{.G_DEPS_DIR}}'" + - "mkdir -p '{{.G_DEPS_CPP_DIR}}'" + - "mkdir -p '{{.G_DEPS_CPP_CHECKSUMS_DIR}}'" - "mkdir -p '{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}'" - - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" - vars: - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" - CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-spider.cmake" - DEP_TASK: "spider-all-parallel" - - task: "utils:combine-cpp-checksum-files" - - spider-all-parallel: - internal: true - run: "once" - deps: - - task: "absl" - - task: "antlr-jar" - - task: "antlr-runtime" - - task: "boost" - - task: "fmt" - - task: "mariadb-connector-cpp" - - task: "msgpack-cxx" - - task: "spdlog" - - task: "ystdlib" - - absl: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_CXX_STANDARD=20" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - LIB_NAME: "absl" - TARBALL_SHA256: "7262daa7c1711406248c10f41026d685e88223bc92817d16fb93c19adb57f669" - TARBALL_URL: "https://github.com/abseil/abseil-cpp/releases/download/20250512.0/\ - abseil-cpp-20250512.0.tar.gz" - - antlr-jar: - internal: true - vars: - CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/antlr-jar.md5" - OUTPUT_FILE: "{{.G_ANTLR_JAR_FILE}}" - run: "once" - deps: - - task: "yscope-dev-utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: - - "{{.OUTPUT_FILE}}" - - task: "utils:init" - cmds: - - task: "yscope-dev-utils:remote:curl" - vars: - FILE_SHA256: "eae2dfa119a64327444672aff63e9ec35a20180dc5b8090b7a6ab85125df4d76" - OUTPUT_FILE: "{{.OUTPUT_FILE}}" - URL: "https://www.antlr.org/download/antlr-{{.G_ANTLR_VERSION}}-complete.jar" - - # This command must be last - - task: "yscope-dev-utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: - - "{{.OUTPUT_FILE}}" - - antlr-runtime: - internal: true - run: "once" - vars: - LIB_NAME: "antlr4-runtime" - - # Paths - BUILD_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-build" - CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" - INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" - SOURCE_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-src" - deps: - - task: "utils:init" - - task: "yscope-dev-utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.INSTALL_PREFIX}}"] - - # NOTE: We can't use `yscope-dev-utils:cmake:install-remote-tar` (and by extension - # `utils:install-remote-cmake-lib`) since it doesn't yet support the `INCLUDE_PATTERNS` and - # `NUM_COMPONENTS_TO_STRIP` parameters. - cmds: - - task: "yscope-dev-utils:remote:download-and-extract-tar" - vars: - FILE_SHA256: "9f18272a9b32b622835a3365f850dd1063d60f5045fb1e12ce475ae6e18a35bb" - INCLUDE_PATTERNS: ["*/runtime/Cpp"] - NUM_COMPONENTS_TO_STRIP: 3 - OUTPUT_DIR: "{{.SOURCE_DIR}}" - URL: "https://github.com/antlr/antlr4/archive/refs/tags/{{.G_ANTLR_VERSION}}.tar.gz" - - task: "yscope-dev-utils:cmake:generate" - vars: - BUILD_DIR: "{{.BUILD_DIR}}" - EXTRA_ARGS: - - "-DANTLR4_INSTALL=ON" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - # Set CMP0135 so that extracted files use the current timestamp as their modification - # timestamp, which ensures the library gets rebuilt if the extracted files change. - - "-DCMAKE_POLICY_DEFAULT_CMP0135=NEW" - SOURCE_DIR: "{{.SOURCE_DIR}}" - - task: "yscope-dev-utils:cmake:build" - vars: - BUILD_DIR: "{{.BUILD_DIR}}" - JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" - - task: "yscope-dev-utils:cmake:install" - vars: - BUILD_DIR: "{{.BUILD_DIR}}" - CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - - # This command must be last - - task: "yscope-dev-utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.INSTALL_PREFIX}}"] - - boost: - internal: true - vars: - VERSION: "1.87.0" - run: "once" - cmds: - - task: "yscope-dev-utils:boost:download-and-install" - vars: - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" - FILE_SHA256: "d6c69e4459eb5d6ec208250291221e7ff4a2affde9af6e49c9303b89c687461f" - TARGETS: - - "filesystem" - - "iostreams" - - "process" - - "program_options" - - "regex" - - "system" - - "url" - URL: "https://github.com/boostorg/boost/releases/download/boost-{{.VERSION}}/\ - boost-{{.VERSION}}-b2-nodocs.tar.gz" - WORK_DIR: "{{.G_DEPS_CPP_DIR}}" - - catch2: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DBUILD_TESTING=OFF" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_CXX_STANDARD=20" - - "-DCMAKE_CXX_STANDARD_REQUIRED=ON" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - LIB_NAME: "Catch2" - TARBALL_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" - TARBALL_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" - - date: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - LIB_NAME: "date" - TARBALL_SHA256: "7a390f200f0ccd207e8cff6757e04817c1a0aec3e327b006b7eb451c57ee3538" - TARBALL_URL: "https://github.com/HowardHinnant/date/archive/refs/tags/v3.0.1.tar.gz" - - fmt: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DFMT_DOC=OFF" - - "-DFMT_TEST=OFF" - LIB_NAME: "{{.G_FMT_LIB_NAME}}" - TARBALL_SHA256: "bc23066d87ab3168f27cef3e97d545fa63314f5c79df5ea444d41d56f962c6af" - TARBALL_URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.2.0.tar.gz" - - liblzma: - internal: true - vars: - COMMON_CMAKE_GEN_ARGS: - - "-DBUILD_TESTING=OFF" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DXZ_DOC=OFF" - - "-DXZ_TOOL_LZMADEC=OFF" - - "-DXZ_TOOL_LZMAINFO=OFF" - - "-DXZ_TOOL_SCRIPTS=OFF" - - "-DXZ_TOOL_SYMLINKS_LZMA=OFF" - - "-DXZ_TOOL_XZ=OFF" - - "-DXZ_TOOL_XZDEC=OFF" - TARBALL_SHA256: "507825b599356c10dca1cd720c9d0d0c9d5400b9de300af00e4d1ea150795543" - TARBALL_URL: "https://github.com/tukaani-project/xz/releases/download/v5.8.1/xz-5.8.1.tar.gz" - run: "once" - deps: - - task: "liblzma-install" - vars: - BUILD_SHARED_LIBS: true - COMMON_CMAKE_GEN_ARGS: - ref: ".COMMON_CMAKE_GEN_ARGS" - TARBALL_SHA256: "{{.TARBALL_SHA256}}" - TARBALL_URL: "{{.TARBALL_URL}}" - - task: "liblzma-install" - vars: - BUILD_SHARED_LIBS: false - COMMON_CMAKE_GEN_ARGS: - ref: ".COMMON_CMAKE_GEN_ARGS" - TARBALL_SHA256: "{{.TARBALL_SHA256}}" - TARBALL_URL: "{{.TARBALL_URL}}" - - liblzma-install: - internal: true - requires: - vars: ["BUILD_SHARED_LIBS", "COMMON_CMAKE_GEN_ARGS", "TARBALL_SHA256", "TARBALL_URL"] - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DBUILD_SHARED_LIBS={{ if .BUILD_SHARED_LIBS }}ON{{ else }}OFF{{ end }}" - - >- - {{ join " " .COMMON_CMAKE_GEN_ARGS }} - LIB_NAME: "LibLZMA-{{ if .BUILD_SHARED_LIBS }}shared{{ else }}static{{ end }}" - TARBALL_SHA256: "{{.TARBALL_SHA256}}" - TARBALL_URL: "{{.TARBALL_URL}}" - - log-surgeon: - internal: true - run: "once" - deps: - - "fmt" - - "microsoft.gsl" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_GSL_LIB_NAME}}.cmake" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-Dlog_surgeon_BUILD_TESTING=OFF" - LIB_NAME: "log_surgeon" - TARBALL_SHA256: "69a99e0804a52c6b6397c5e7eabecc9bb4915d0145632c66fc63ad13678ff56a" - TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/a722d07.tar.gz" - - lz4: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DBUILD_SHARED_LIBS=ON" - - "-DBUILD_STATIC_LIBS=ON" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DLZ4_BUILD_CLI=OFF" - CMAKE_SOURCE_DIR: "build/cmake" - LIB_NAME: "lz4" - TARBALL_SHA256: "537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b" - TARBALL_URL: "https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz" - - mariadb-connector-cpp: - platforms: ["linux"] - preconditions: - # NOTE: The MariaDB connector is only required for building Spider, which is only supported on - # Ubuntu. `INSTALL_LAYOUT` is currently hardcoded to "DEB". To support other Linux distros: - # - Update the precondition accordingly. - # - Make `INSTALL_LAYOUT` configurable. - - >- - source /etc/os-release && [[ "$ID" == "ubuntu" ]] - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - - "-DINSTALL_LAYOUT=DEB" - LIB_NAME: "mariadb-connector-cpp" - TARBALL_URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/\ - tags/1.1.5.tar.gz" - TARBALL_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" - - microsoft.gsl: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DGSL_TEST=OFF" - LIB_NAME: "{{.G_GSL_LIB_NAME}}" - TARBALL_SHA256: "f0e32cb10654fea91ad56bde89170d78cfbf4363ee0b01d8f097de2ba49f6ce9" - TARBALL_URL: "https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.tar.gz" - - mongocxx: - internal: true - vars: - VERSION: "r4.1.1" - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DBUILD_SHARED_AND_STATIC_LIBS=ON" - - "-DBUILD_SHARED_LIBS_WITH_STATIC_MONGOC=ON" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DENABLE_UNINSTALL=OFF" - LIB_NAME: "mongocxx" - TARBALL_SHA256: "19dff3cf834a3e09229260f22a0325820a7e30c78b294db91794dd934776b33a" - TARBALL_URL: "https://github.com/mongodb/mongo-cxx-driver/releases/download/{{.VERSION}}\ - /mongo-cxx-driver-{{.VERSION}}.tar.gz" - - msgpack-cxx: - internal: true - vars: - VERSION: "7.0.0" - run: "once" - deps: - - task: "boost" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" - - "-DMSGPACK_BUILD_DOCS=OFF" - - "-DMSGPACK_CXX20=ON" - LIB_NAME: "msgpack-cxx" - TARBALL_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" - TARBALL_URL: "https://github.com/msgpack/msgpack-c/releases/download/cpp-{{.VERSION}}\ - /msgpack-cxx-{{.VERSION}}.tar.gz" - - nlohmann_json: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DJSON_BuildTests=OFF" - LIB_NAME: "nlohmann_json" - TARBALL_SHA256: "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406" - - # NOTE: We use the GitHub-generated source tarball for this version rather than the - # release tarball, since the latter is served from githubusercontent.com which is blocked - # by some developers' firewalls. The contents of the former are a superset of the latter. - TARBALL_URL: "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz" - - simdjson: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - LIB_NAME: "simdjson" - TARBALL_SHA256: "07a1bb3587aac18fd6a10a83fe4ab09f1100ab39f0cb73baea1317826b9f9e0d" - TARBALL_URL: "https://github.com/simdjson/simdjson/archive/refs/tags/v3.13.0.tar.gz" - - spdlog: - internal: true - run: "once" - deps: - - "fmt" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" - - "-DSPDLOG_BUILD_EXAMPLE=OFF" - - "-DSPDLOG_BUILD_EXAMPLE_HO=OFF" - - "-DSPDLOG_FMT_EXTERNAL=ON" - LIB_NAME: "spdlog" - TARBALL_SHA256: "15a04e69c222eb6c01094b5c7ff8a249b36bb22788d72519646fb85feb267e67" - - # NOTE: Since spdlog depends on fmt, we need to choose a version of spdlog that's - # compatible with the version of fmt we use. - TARBALL_URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.3.tar.gz" - - sqlite3: - internal: true - vars: - LIB_NAME: "sqlite3" - - # Paths - EXTRACTION_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-src" - INSTALL_INCLUDE_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install/include" - INSTALL_SYMLINK: "{{.INSTALL_INCLUDE_DIR}}/{{.LIB_NAME}}" - ZIP_FILENAME_STEM: "sqlite-amalgamation-3360000" - SRC_DIR: "{{.EXTRACTION_DIR}}/{{.ZIP_FILENAME_STEM}}" - run: "once" - deps: - - task: "utils:init" - cmds: - - task: "yscope-dev-utils:remote:download-and-extract-zip" - vars: - CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" - FILE_SHA256: "999826fe4c871f18919fdb8ed7ec9dd8217180854dd1fe21eea96aed36186729" - OUTPUT_DIR: "{{.EXTRACTION_DIR}}" - URL: "https://www.sqlite.org/2021/{{.ZIP_FILENAME_STEM}}.zip" - - "mkdir -p '{{.INSTALL_INCLUDE_DIR}}'" - - "rm -f '{{.INSTALL_SYMLINK}}'" - - "ln -s '{{.SRC_DIR}}' '{{.INSTALL_SYMLINK}}'" - - |- - echo "set(CLP_SQLITE3_SOURCE_DIRECTORY \"{{.SRC_DIR}}\")" \ - > "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" - echo "set(CLP_SQLITE3_INCLUDE_DIRECTORY \"{{.INSTALL_INCLUDE_DIR}}\")" \ - >> "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" - - utfcpp: - internal: true - vars: - LIB_NAME: "utfcpp" - UTFCPP_OUTPUT_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-src" - run: "once" - deps: - - task: "utils:init" - cmds: - - task: "yscope-dev-utils:remote:download-and-extract-tar" - vars: - CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" - FILE_SHA256: "6920a6a5d6a04b9a89b2a89af7132f8acefd46e0c2a7b190350539e9213816c0" - OUTPUT_DIR: "{{.UTFCPP_OUTPUT_DIR}}" - URL: "https://github.com/nemtrif/utfcpp/archive/refs/tags/v4.0.6.tar.gz" - - >- - echo "set( - CLP_UTFCPP_INCLUDE_DIRECTORY \"{{.UTFCPP_OUTPUT_DIR}}\" - )" > "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" - - yaml-cpp: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DBUILD_TESTING=OFF" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - LIB_NAME: "yaml-cpp" - TARBALL_SHA256: "43e6a9fcb146ad871515f0d0873947e5d497a1c9c60c58cb102a97b47208b7c3" - TARBALL_URL: "https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.tar.gz" - - ystdlib: - internal: true - run: "once" - deps: - - task: "boost" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" - - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - - "-Dystdlib_BUILD_TESTING=OFF" - LIB_NAME: "ystdlib" - TARBALL_SHA256: "65990dc2bcc4a355c2181bfe31a7800f492309d1bcd340f52a34e85047e61bc8" - TARBALL_URL: "https://github.com/y-scope/ystdlib-cpp/archive/9ed78cd.tar.gz" - - zlib: - internal: true - vars: - LIB_NAME: "zlib" - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DBUILD_SHARED_LIBS=ON" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DCMAKE_INSTALL_PREFIX={{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" - - "-DZLIB_BUILD_EXAMPLES=OFF" - LIB_NAME: "{{.LIB_NAME}}" - TARBALL_SHA256: "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23" - TARBALL_URL: "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz" - - zstd: - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DZSTD_BUILD_CONTRIB=OFF" - - "-DZSTD_BUILD_PROGRAMS=OFF" - - "-DZSTD_BUILD_SHARED=ON" - - "-DZSTD_BUILD_STATIC=ON" - - "-DZSTD_BUILD_TESTS=OFF" - CMAKE_SOURCE_DIR: "build/cmake" - LIB_NAME: "zstd" - TARBALL_SHA256: "eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" - TARBALL_URL: "https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz" diff --git a/taskfiles/deps/metadata.yaml b/taskfiles/deps/metadata.yaml new file mode 100644 index 0000000000..1db4764c5d --- /dev/null +++ b/taskfiles/deps/metadata.yaml @@ -0,0 +1,28 @@ +version: "3" + +vars: + G_DEPS_LIB_NAMES: + map: + absl: "absl" + antlr-runtime: "antlr4-runtime" + boost: "Boost" + catch2: "Catch2" + date: "date" + fmt: "fmt" + liblzma: "LibLZMA" + log-surgeon: "log_surgeon" + lz4: "lz4" + mariadb-connector-cpp: "mariadb-connector-cpp" + microsoft.gsl: "Microsoft.GSL" + mongocxx: "mongocxx" + msgpack-cxx: "msgpack-cxx" + nlohmann_json: "nlohmann_json" + simdjson: "simdjson" + spdlog: "spdlog" + spider: "spider" + sqlite3: "sqlite3" + utfcpp: "utfcpp" + yaml-cpp: "yaml-cpp" + ystdlib: "ystdlib" + zlib: "zlib" + zstd: "zstd" diff --git a/taskfiles/deps/misc-libs.yaml b/taskfiles/deps/misc-libs.yaml new file mode 100644 index 0000000000..2588994662 --- /dev/null +++ b/taskfiles/deps/misc-libs.yaml @@ -0,0 +1,136 @@ +version: "3" + +tasks: + antlr-jar: + internal: true + vars: + CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/antlr-jar.md5" + OUTPUT_FILE: "{{.G_ANTLR_JAR_FILE}}" + run: "once" + deps: + - task: "yscope-dev-utils:checksum:validate" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: + - "{{.OUTPUT_FILE}}" + cmds: + - task: "yscope-dev-utils:remote:curl" + vars: + FILE_SHA256: "eae2dfa119a64327444672aff63e9ec35a20180dc5b8090b7a6ab85125df4d76" + OUTPUT_FILE: "{{.OUTPUT_FILE}}" + URL: "https://www.antlr.org/download/antlr-{{.G_ANTLR_VERSION}}-complete.jar" + + # This command must be last + - task: "yscope-dev-utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: + - "{{.OUTPUT_FILE}}" + + antlr-runtime: + internal: true + run: "once" + vars: + LIB_NAME: "antlr4-runtime" + + # Paths + BUILD_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-build" + CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" + INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" + SOURCE_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-src" + deps: + - task: "yscope-dev-utils:checksum:validate" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.INSTALL_PREFIX}}"] + + # NOTE: We can't use `yscope-dev-utils:cmake:install-remote-tar` (and by extension + # `utils:install-remote-cmake-lib`) since it doesn't yet support the `INCLUDE_PATTERNS` and + # `NUM_COMPONENTS_TO_STRIP` parameters. + cmds: + - task: "yscope-dev-utils:remote:download-and-extract-tar" + vars: + FILE_SHA256: "9f18272a9b32b622835a3365f850dd1063d60f5045fb1e12ce475ae6e18a35bb" + INCLUDE_PATTERNS: ["*/runtime/Cpp"] + NUM_COMPONENTS_TO_STRIP: 3 + OUTPUT_DIR: "{{.SOURCE_DIR}}" + URL: "https://github.com/antlr/antlr4/archive/refs/tags/{{.G_ANTLR_VERSION}}.tar.gz" + - task: "yscope-dev-utils:cmake:generate" + vars: + BUILD_DIR: "{{.BUILD_DIR}}" + EXTRA_ARGS: + - "-DANTLR4_INSTALL=ON" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + + # Set CMP0135 so that extracted files use the current timestamp as their modification + # timestamp, which ensures the library gets rebuilt if the extracted files change. + - "-DCMAKE_POLICY_DEFAULT_CMP0135=NEW" + SOURCE_DIR: "{{.SOURCE_DIR}}" + - task: "yscope-dev-utils:cmake:build" + vars: + BUILD_DIR: "{{.BUILD_DIR}}" + JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" + - task: "yscope-dev-utils:cmake:install" + vars: + BUILD_DIR: "{{.BUILD_DIR}}" + CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" + + # This command must be last + - task: "yscope-dev-utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.INSTALL_PREFIX}}"] + + boost: + internal: true + vars: + VERSION: "1.87.0" + run: "once" + cmds: + - task: "yscope-dev-utils:boost:download-and-install" + vars: + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + FILE_SHA256: "d6c69e4459eb5d6ec208250291221e7ff4a2affde9af6e49c9303b89c687461f" + TARGETS: + - "filesystem" + - "iostreams" + - "process" + - "program_options" + - "regex" + - "system" + - "url" + URL: "https://github.com/boostorg/boost/releases/download/boost-{{.VERSION}}/\ + boost-{{.VERSION}}-b2-nodocs.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + + sqlite3: + internal: true + vars: + LIB_NAME: "sqlite3" + + # Paths + EXTRACTION_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-src" + INSTALL_INCLUDE_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install/include" + INSTALL_SYMLINK: "{{.INSTALL_INCLUDE_DIR}}/{{.LIB_NAME}}" + ZIP_FILENAME_STEM: "sqlite-amalgamation-3360000" + SRC_DIR: "{{.EXTRACTION_DIR}}/{{.ZIP_FILENAME_STEM}}" + run: "once" + cmds: + - task: "yscope-dev-utils:remote:download-and-extract-zip" + vars: + CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" + FILE_SHA256: "999826fe4c871f18919fdb8ed7ec9dd8217180854dd1fe21eea96aed36186729" + OUTPUT_DIR: "{{.EXTRACTION_DIR}}" + URL: "https://www.sqlite.org/2021/{{.ZIP_FILENAME_STEM}}.zip" + - "mkdir -p '{{.INSTALL_INCLUDE_DIR}}'" + - "rm -f '{{.INSTALL_SYMLINK}}'" + - "ln -s '{{.SRC_DIR}}' '{{.INSTALL_SYMLINK}}'" + - |- + echo "set(CLP_SQLITE3_SOURCE_DIRECTORY \"{{.SRC_DIR}}\")" \ + > "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" + echo "set(CLP_SQLITE3_INCLUDE_DIRECTORY \"{{.INSTALL_INCLUDE_DIR}}\")" \ + >> "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" diff --git a/taskfiles/deps/spider.yaml b/taskfiles/deps/spider.yaml new file mode 100644 index 0000000000..e95f70abb9 --- /dev/null +++ b/taskfiles/deps/spider.yaml @@ -0,0 +1,55 @@ +version: "3" + +tasks: + spider: + deps: + - task: "spider-dep" + platforms: ["linux"] + preconditions: + # NOTE: Spider is only supported on Ubuntu at present. + - >- + source /etc/os-release && [[ "$ID" == "ubuntu" ]] + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_PACKAGE_NAME: "spider" + TAR_SHA256: "43478b6da99aa2df50c960573ec68c6fcf67cdfbab39b5e314676218aa93b959" + TAR_URL: "https://github.com/y-scope/spider/archive/6a7bad3.tar.gz" + WORK_DIR: "{{.G_SPIDER_BUILD_DIR}}" + CMAKE_JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-spider.cmake" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DSPIDER_BUILD_TESTING=OFF" + + spider-dep: + deps: + - task: "init" + - task: "utils:clean-outdated-cpp-checksum-files" + cmds: + - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" + vars: + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-spider.cmake" + DEP_TASK: "spider-all-parallel" + - task: "utils:combine-cpp-checksum-files" + + spider-all-parallel: + internal: true + vars: + CPP_SPIDER_TASKS: + - "absl" + - "antlr-jar" + - "antlr-runtime" + - "boost" + - "fmt" + - "mariadb-connector-cpp" + - "msgpack-cxx" + - "spdlog" + - "ystdlib" + run: "once" + deps: + - for: + var: "CPP_SPIDER_TASKS" + task: "{{.ITEM}}" diff --git a/taskfiles/deps/utils.yaml b/taskfiles/deps/utils.yaml index b6f71720a1..ccd17883f0 100644 --- a/taskfiles/deps/utils.yaml +++ b/taskfiles/deps/utils.yaml @@ -29,15 +29,6 @@ tasks: fi done - init: - internal: true - run: "once" - cmds: - - "mkdir -p '{{.G_DEPS_DIR}}'" - - "mkdir -p '{{.G_DEPS_CPP_DIR}}'" - - "mkdir -p '{{.G_DEPS_CPP_CHECKSUMS_DIR}}'" - - "mkdir -p '{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}'" - # Installs a CMake-based library from a remote tarball, generates a corresponding settings file, # and computes a checksum for the installation directory. # @@ -58,7 +49,6 @@ tasks: INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" INSTALL_DIR_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" deps: - - task: "init" - task: "yscope-dev-utils:checksum:validate" vars: CHECKSUM_FILE: "{{.INSTALL_DIR_CHECKSUM_FILE}}" From 5abd02dbd3042e40c685342d6087d088e2e3b52c Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 4 Oct 2025 11:55:51 -0400 Subject: [PATCH 02/13] Remove unnecessary flattening of dep task files --- taskfile.yaml | 2 +- taskfiles/deps/cmake-libs.yaml | 6 ++++++ taskfiles/deps/core.yaml | 13 +++++++++++++ taskfiles/deps/log-viewer.yaml | 5 ++++- taskfiles/deps/main.yaml | 29 +++-------------------------- taskfiles/deps/misc-libs.yaml | 5 +++++ taskfiles/deps/spider.yaml | 4 ++++ 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index f1873b9d91..5055f7e7b2 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -122,7 +122,7 @@ tasks: - "core" - "clp-package-utils" - "clp-py-utils" - - "deps:spider" + - "deps:spider:spider-dep" - "init" - "job-orchestration" - "package-venv" diff --git a/taskfiles/deps/cmake-libs.yaml b/taskfiles/deps/cmake-libs.yaml index 5805991435..0fcda9f698 100644 --- a/taskfiles/deps/cmake-libs.yaml +++ b/taskfiles/deps/cmake-libs.yaml @@ -1,5 +1,11 @@ version: "3" +vars: + # Library names + G_BOOST_LIB_NAME: "Boost" + G_FMT_LIB_NAME: "fmt" + G_GSL_LIB_NAME: "Microsoft.GSL" + # NOTE: For dependencies built using CMake, we set CMP0074 to NEW whenever: # - a component's minimum required CMake version is less than 3.27 (where CMP0074 defaults to OLD); # and diff --git a/taskfiles/deps/core.yaml b/taskfiles/deps/core.yaml index 475024e21a..326ad297fb 100644 --- a/taskfiles/deps/core.yaml +++ b/taskfiles/deps/core.yaml @@ -1,6 +1,19 @@ version: "3" +vars: + # Checksum paths + G_DEPS_CPP_CHECKSUMS_DIR: "{{.G_DEPS_CPP_DIR}}/checksums" + G_DEPS_CPP_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/all.md5" + + # CMake settings paths + # NOTE: This must be kept in-sync with its usage in components/core/CMakeLists.txt + G_DEPS_CPP_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_DIR}}/cmake-settings" + tasks: + default: + deps: + - task: "core" + core: deps: - task: "init" diff --git a/taskfiles/deps/log-viewer.yaml b/taskfiles/deps/log-viewer.yaml index 10d623ef4e..cc8fece66a 100644 --- a/taskfiles/deps/log-viewer.yaml +++ b/taskfiles/deps/log-viewer.yaml @@ -1,7 +1,10 @@ version: "3" +vars: + G_DEPS_LOG_VIEWER_CHECKSUM_FILE: "{{.G_DEPS_DIR}}/log-viewer.md5" + tasks: - log-viewer: + default: deps: - task: "init" cmds: diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 663295a63b..6fae9d9e4e 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -4,18 +4,12 @@ includes: cmake-libs: flatten: true taskfile: "cmake-libs.yaml" - core: - flatten: true - taskfile: "core.yaml" + core: "core.yaml" misc-libs: flatten: true taskfile: "misc-libs.yaml" - spider: - flatten: true - taskfile: "spider.yaml" - log-viewer: - flatten: true - taskfile: "log-viewer.yaml" + spider: "spider.yaml" + log-viewer: "log-viewer.yaml" utils: "utils.yaml" yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" @@ -23,23 +17,6 @@ vars: # Directories G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" G_DEPS_CPP_DIR: "{{.G_DEPS_DIR}}/cpp" - G_DEPS_CPP_CHECKSUMS_DIR: "{{.G_DEPS_CPP_DIR}}/checksums" - - # Checksum file paths - G_DEPS_CPP_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/all.md5" - G_DEPS_LOG_VIEWER_CHECKSUM_FILE: "{{.G_DEPS_DIR}}/log-viewer.md5" - - # NOTE: This must be kept in-sync with its usage in components/core/CMakeLists.txt - G_DEPS_CPP_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_DIR}}/cmake-settings" - - # Library names - G_BOOST_LIB_NAME: "Boost" - G_FMT_LIB_NAME: "fmt" - G_GSL_LIB_NAME: "Microsoft.GSL" - - # Antlr - G_ANTLR_VERSION: "4.13.2" - G_ANTLR_JAR_FILE: "{{.G_DEPS_CPP_DIR}}/antlr-{{.G_ANTLR_VERSION}}-complete.jar" tasks: default: diff --git a/taskfiles/deps/misc-libs.yaml b/taskfiles/deps/misc-libs.yaml index 2588994662..39668a998a 100644 --- a/taskfiles/deps/misc-libs.yaml +++ b/taskfiles/deps/misc-libs.yaml @@ -1,5 +1,10 @@ version: "3" +vars: + # Antlr + G_ANTLR_VERSION: "4.13.2" + G_ANTLR_JAR_FILE: "{{.G_DEPS_CPP_DIR}}/antlr-{{.G_ANTLR_VERSION}}-complete.jar" + tasks: antlr-jar: internal: true diff --git a/taskfiles/deps/spider.yaml b/taskfiles/deps/spider.yaml index e95f70abb9..798539d3bb 100644 --- a/taskfiles/deps/spider.yaml +++ b/taskfiles/deps/spider.yaml @@ -1,6 +1,10 @@ version: "3" tasks: + default: + deps: + - task: "spider" + spider: deps: - task: "spider-dep" From 57f72d7ab842eda20666214a01717a6908433810 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 4 Oct 2025 11:56:20 -0400 Subject: [PATCH 03/13] remove unused metadata file --- taskfiles/deps/metadata.yaml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 taskfiles/deps/metadata.yaml diff --git a/taskfiles/deps/metadata.yaml b/taskfiles/deps/metadata.yaml deleted file mode 100644 index 1db4764c5d..0000000000 --- a/taskfiles/deps/metadata.yaml +++ /dev/null @@ -1,28 +0,0 @@ -version: "3" - -vars: - G_DEPS_LIB_NAMES: - map: - absl: "absl" - antlr-runtime: "antlr4-runtime" - boost: "Boost" - catch2: "Catch2" - date: "date" - fmt: "fmt" - liblzma: "LibLZMA" - log-surgeon: "log_surgeon" - lz4: "lz4" - mariadb-connector-cpp: "mariadb-connector-cpp" - microsoft.gsl: "Microsoft.GSL" - mongocxx: "mongocxx" - msgpack-cxx: "msgpack-cxx" - nlohmann_json: "nlohmann_json" - simdjson: "simdjson" - spdlog: "spdlog" - spider: "spider" - sqlite3: "sqlite3" - utfcpp: "utfcpp" - yaml-cpp: "yaml-cpp" - ystdlib: "ystdlib" - zlib: "zlib" - zstd: "zstd" From 2f1addbe96c2affa7a9d5349db30b45b60e1ce6e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 4 Oct 2025 11:59:01 -0400 Subject: [PATCH 04/13] Fix init task namespaces --- taskfiles/deps/core.yaml | 2 +- taskfiles/deps/log-viewer.yaml | 2 +- taskfiles/deps/spider.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/taskfiles/deps/core.yaml b/taskfiles/deps/core.yaml index 326ad297fb..52a5b3a8f8 100644 --- a/taskfiles/deps/core.yaml +++ b/taskfiles/deps/core.yaml @@ -16,7 +16,7 @@ tasks: core: deps: - - task: "init" + - task: ":init" - task: "utils:clean-outdated-cpp-checksum-files" cmds: - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" diff --git a/taskfiles/deps/log-viewer.yaml b/taskfiles/deps/log-viewer.yaml index cc8fece66a..63b6481d19 100644 --- a/taskfiles/deps/log-viewer.yaml +++ b/taskfiles/deps/log-viewer.yaml @@ -6,7 +6,7 @@ vars: tasks: default: deps: - - task: "init" + - task: ":init" cmds: - task: "yscope-dev-utils:remote:download-and-extract-tar" vars: diff --git a/taskfiles/deps/spider.yaml b/taskfiles/deps/spider.yaml index 798539d3bb..134870633d 100644 --- a/taskfiles/deps/spider.yaml +++ b/taskfiles/deps/spider.yaml @@ -29,7 +29,7 @@ tasks: spider-dep: deps: - - task: "init" + - task: ":init" - task: "utils:clean-outdated-cpp-checksum-files" cmds: - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" From 6a0b11665d91bbbf68f3ac6ca1d72b5e8fb99928 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sat, 4 Oct 2025 12:09:31 -0400 Subject: [PATCH 05/13] use proper includes --- taskfiles/deps/core.yaml | 10 ++++++++++ taskfiles/deps/log-viewer.yaml | 10 ++++++++++ taskfiles/deps/main.yaml | 6 ------ taskfiles/deps/spider.yaml | 10 ++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/taskfiles/deps/core.yaml b/taskfiles/deps/core.yaml index 52a5b3a8f8..364e4b83c8 100644 --- a/taskfiles/deps/core.yaml +++ b/taskfiles/deps/core.yaml @@ -1,5 +1,15 @@ version: "3" +includes: + cmake-libs: + flatten: true + taskfile: "cmake-libs.yaml" + misc-libs: + flatten: true + taskfile: "misc-libs.yaml" + utils: "utils.yaml" + yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" + vars: # Checksum paths G_DEPS_CPP_CHECKSUMS_DIR: "{{.G_DEPS_CPP_DIR}}/checksums" diff --git a/taskfiles/deps/log-viewer.yaml b/taskfiles/deps/log-viewer.yaml index 63b6481d19..6d9d69adc5 100644 --- a/taskfiles/deps/log-viewer.yaml +++ b/taskfiles/deps/log-viewer.yaml @@ -1,5 +1,15 @@ version: "3" +includes: + cmake-libs: + flatten: true + taskfile: "cmake-libs.yaml" + misc-libs: + flatten: true + taskfile: "misc-libs.yaml" + utils: "utils.yaml" + yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" + vars: G_DEPS_LOG_VIEWER_CHECKSUM_FILE: "{{.G_DEPS_DIR}}/log-viewer.md5" diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 6fae9d9e4e..3c4a2ac791 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -1,13 +1,7 @@ version: "3" includes: - cmake-libs: - flatten: true - taskfile: "cmake-libs.yaml" core: "core.yaml" - misc-libs: - flatten: true - taskfile: "misc-libs.yaml" spider: "spider.yaml" log-viewer: "log-viewer.yaml" utils: "utils.yaml" diff --git a/taskfiles/deps/spider.yaml b/taskfiles/deps/spider.yaml index 134870633d..3656f5ea0b 100644 --- a/taskfiles/deps/spider.yaml +++ b/taskfiles/deps/spider.yaml @@ -1,5 +1,15 @@ version: "3" +includes: + cmake-libs: + flatten: true + taskfile: "cmake-libs.yaml" + misc-libs: + flatten: true + taskfile: "misc-libs.yaml" + utils: "utils.yaml" + yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" + tasks: default: deps: From a80214686ec704e30a44cd7107e9a638210e19cb Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 5 Oct 2025 05:10:13 -0400 Subject: [PATCH 06/13] Properly design taskfile includes --- taskfiles/deps/core.yaml | 29 +++++------------------------ taskfiles/deps/log-viewer.yaml | 12 +----------- taskfiles/deps/main.yaml | 19 +++++++++++++++++++ taskfiles/deps/misc-libs.yaml | 2 +- taskfiles/deps/spider.yaml | 22 ++++++---------------- taskfiles/deps/utils.yaml | 11 ++++------- 6 files changed, 36 insertions(+), 59 deletions(-) diff --git a/taskfiles/deps/core.yaml b/taskfiles/deps/core.yaml index 364e4b83c8..c3f07c3725 100644 --- a/taskfiles/deps/core.yaml +++ b/taskfiles/deps/core.yaml @@ -1,24 +1,5 @@ version: "3" -includes: - cmake-libs: - flatten: true - taskfile: "cmake-libs.yaml" - misc-libs: - flatten: true - taskfile: "misc-libs.yaml" - utils: "utils.yaml" - yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" - -vars: - # Checksum paths - G_DEPS_CPP_CHECKSUMS_DIR: "{{.G_DEPS_CPP_DIR}}/checksums" - G_DEPS_CPP_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/all.md5" - - # CMake settings paths - # NOTE: This must be kept in-sync with its usage in components/core/CMakeLists.txt - G_DEPS_CPP_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_DIR}}/cmake-settings" - tasks: default: deps: @@ -27,14 +8,14 @@ tasks: core: deps: - task: ":init" - - task: "utils:clean-outdated-cpp-checksum-files" + - task: ":utils:clean-outdated-cpp-checksum-files" cmds: - - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" + - task: ":yscope-dev-utils:cmake:install-deps-and-generate-settings" vars: CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-core.cmake" - DEP_TASK: "core-all-parallel" - - task: "utils:combine-cpp-checksum-files" + DEP_TASK: "core:core-all-parallel" + - task: ":utils:combine-cpp-checksum-files" core-all-parallel: internal: true @@ -66,4 +47,4 @@ tasks: deps: - for: var: "CPP_CORE_TASKS" - task: "{{.ITEM}}" + task: ":{{.ITEM}}" diff --git a/taskfiles/deps/log-viewer.yaml b/taskfiles/deps/log-viewer.yaml index 6d9d69adc5..d7c2ac8baa 100644 --- a/taskfiles/deps/log-viewer.yaml +++ b/taskfiles/deps/log-viewer.yaml @@ -1,15 +1,5 @@ version: "3" -includes: - cmake-libs: - flatten: true - taskfile: "cmake-libs.yaml" - misc-libs: - flatten: true - taskfile: "misc-libs.yaml" - utils: "utils.yaml" - yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" - vars: G_DEPS_LOG_VIEWER_CHECKSUM_FILE: "{{.G_DEPS_DIR}}/log-viewer.md5" @@ -18,7 +8,7 @@ tasks: deps: - task: ":init" cmds: - - task: "yscope-dev-utils:remote:download-and-extract-tar" + - task: ":yscope-dev-utils:remote:download-and-extract-tar" vars: CHECKSUM_FILE: "{{.G_DEPS_LOG_VIEWER_CHECKSUM_FILE}}" FILE_SHA256: "f9baaae0886439862204e8062c69dd49354cb14ad4e59f7c35779d544384955e" diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 3c4a2ac791..4ef19cd02c 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -1,17 +1,36 @@ version: "3" includes: + # Components core: "core.yaml" spider: "spider.yaml" log-viewer: "log-viewer.yaml" + + # Utils utils: "utils.yaml" yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" + # Individual library installs + cmake-libs: + flatten: true + taskfile: "cmake-libs.yaml" + misc-libs: + flatten: true + taskfile: "misc-libs.yaml" + vars: # Directories G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" G_DEPS_CPP_DIR: "{{.G_DEPS_DIR}}/cpp" + # Checksum dirs and paths + G_DEPS_CPP_CHECKSUMS_DIR: "{{.G_DEPS_CPP_DIR}}/checksums" + G_DEPS_CPP_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/all.md5" + + # CMake settings paths + # NOTE: This must be kept in-sync with its usage in components/core/CMakeLists.txt + G_DEPS_CPP_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_DIR}}/cmake-settings" + tasks: default: deps: diff --git a/taskfiles/deps/misc-libs.yaml b/taskfiles/deps/misc-libs.yaml index 39668a998a..e09b34b6f4 100644 --- a/taskfiles/deps/misc-libs.yaml +++ b/taskfiles/deps/misc-libs.yaml @@ -34,7 +34,6 @@ tasks: antlr-runtime: internal: true - run: "once" vars: LIB_NAME: "antlr4-runtime" @@ -43,6 +42,7 @@ tasks: CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" SOURCE_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-src" + run: "once" deps: - task: "yscope-dev-utils:checksum:validate" vars: diff --git a/taskfiles/deps/spider.yaml b/taskfiles/deps/spider.yaml index 3656f5ea0b..ce241fb9e2 100644 --- a/taskfiles/deps/spider.yaml +++ b/taskfiles/deps/spider.yaml @@ -1,15 +1,5 @@ version: "3" -includes: - cmake-libs: - flatten: true - taskfile: "cmake-libs.yaml" - misc-libs: - flatten: true - taskfile: "misc-libs.yaml" - utils: "utils.yaml" - yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" - tasks: default: deps: @@ -24,7 +14,7 @@ tasks: - >- source /etc/os-release && [[ "$ID" == "ubuntu" ]] cmds: - - task: "yscope-dev-utils:cmake:install-remote-tar" + - task: ":yscope-dev-utils:cmake:install-remote-tar" vars: CMAKE_PACKAGE_NAME: "spider" TAR_SHA256: "43478b6da99aa2df50c960573ec68c6fcf67cdfbab39b5e314676218aa93b959" @@ -40,14 +30,14 @@ tasks: spider-dep: deps: - task: ":init" - - task: "utils:clean-outdated-cpp-checksum-files" + - task: ":utils:clean-outdated-cpp-checksum-files" cmds: - - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" + - task: ":yscope-dev-utils:cmake:install-deps-and-generate-settings" vars: CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-spider.cmake" - DEP_TASK: "spider-all-parallel" - - task: "utils:combine-cpp-checksum-files" + DEP_TASK: "spider:spider-all-parallel" + - task: ":utils:combine-cpp-checksum-files" spider-all-parallel: internal: true @@ -66,4 +56,4 @@ tasks: deps: - for: var: "CPP_SPIDER_TASKS" - task: "{{.ITEM}}" + task: ":{{.ITEM}}" diff --git a/taskfiles/deps/utils.yaml b/taskfiles/deps/utils.yaml index ccd17883f0..847f2410d2 100644 --- a/taskfiles/deps/utils.yaml +++ b/taskfiles/deps/utils.yaml @@ -1,8 +1,5 @@ version: "3" -includes: - yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" - tasks: # Cleans up the cpp checksum files any time the taskfiles change. This is to ensure that if any # of cpp's dependencies are removed from the taskfiles, then the checksum files for the removed @@ -10,11 +7,11 @@ tasks: # removed dependencies, we remove all of them. clean-outdated-cpp-checksum-files: internal: true - run: "once" sources: - "{{.ROOT_DIR}}/Taskfile.yaml" - "{{.ROOT_DIR}}/taskfiles/**/*.yaml" - "{{.ROOT_DIR}}/tools/yscope-dev-utils/exports/taskfiles/**/*.yaml" + run: "once" cmd: "rm -f '{{.G_DEPS_CPP_CHECKSUMS_DIR}}'/*" # Combines all of cpp's checksum files into `G_DEPS_CPP_CHECKSUM_FILE`. @@ -49,13 +46,13 @@ tasks: INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" INSTALL_DIR_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" deps: - - task: "yscope-dev-utils:checksum:validate" + - task: ":yscope-dev-utils:checksum:validate" vars: CHECKSUM_FILE: "{{.INSTALL_DIR_CHECKSUM_FILE}}" INCLUDE_PATTERNS: - "{{.INSTALL_PREFIX}}" cmds: - - task: "yscope-dev-utils:cmake:install-remote-tar" + - task: ":yscope-dev-utils:cmake:install-remote-tar" vars: CMAKE_GEN_ARGS: ref: ".CMAKE_GEN_ARGS" @@ -68,7 +65,7 @@ tasks: WORK_DIR: "{{.G_DEPS_CPP_DIR}}" # This command must be last - - task: "yscope-dev-utils:checksum:compute" + - task: ":yscope-dev-utils:checksum:compute" vars: CHECKSUM_FILE: "{{.INSTALL_DIR_CHECKSUM_FILE}}" INCLUDE_PATTERNS: From 9160b55f5f68c475599a42c595d8cccac782babf Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 5 Oct 2025 05:23:12 -0400 Subject: [PATCH 07/13] misc reorder of mariadb-connector-cpp task entries according to taskfile guidlines --- taskfiles/deps/cmake-libs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfiles/deps/cmake-libs.yaml b/taskfiles/deps/cmake-libs.yaml index 0fcda9f698..25cb3da5f7 100644 --- a/taskfiles/deps/cmake-libs.yaml +++ b/taskfiles/deps/cmake-libs.yaml @@ -155,6 +155,7 @@ tasks: TARBALL_URL: "https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz" mariadb-connector-cpp: + internal: true platforms: ["linux"] preconditions: # NOTE: The MariaDB connector is only required for building Spider, which is only supported on @@ -163,7 +164,6 @@ tasks: # - Make `INSTALL_LAYOUT` configurable. - >- source /etc/os-release && [[ "$ID" == "ubuntu" ]] - internal: true run: "once" cmds: - task: "utils:install-remote-cmake-lib" @@ -172,9 +172,9 @@ tasks: - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=DEB" LIB_NAME: "mariadb-connector-cpp" + TARBALL_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" TARBALL_URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/\ tags/1.1.5.tar.gz" - TARBALL_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" microsoft.gsl: internal: true From edba092727a7322ae9db272fb029411176f4707e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 6 Oct 2025 12:16:13 -0400 Subject: [PATCH 08/13] properly fix cmake-settings --- taskfiles/deps/cmake-libs.yaml | 152 +++++++++++++++++++--------- taskfiles/deps/core.yaml | 11 +- taskfiles/deps/main.yaml | 6 +- taskfiles/deps/misc-libs.yaml | 31 ++++-- taskfiles/deps/spider.yaml | 13 ++- taskfiles/deps/utils.yaml | 42 +++++++- tools/scripts/deps-download/init.sh | 4 +- tools/yscope-dev-utils | 2 +- 8 files changed, 194 insertions(+), 67 deletions(-) diff --git a/taskfiles/deps/cmake-libs.yaml b/taskfiles/deps/cmake-libs.yaml index 25cb3da5f7..b4e1bae4ea 100644 --- a/taskfiles/deps/cmake-libs.yaml +++ b/taskfiles/deps/cmake-libs.yaml @@ -13,14 +13,16 @@ vars: tasks: absl: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_CXX_STANDARD=20" - "-DCMAKE_INSTALL_MESSAGE=LAZY" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "absl" TARBALL_SHA256: "7262daa7c1711406248c10f41026d685e88223bc92817d16fb93c19adb57f669" TARBALL_URL: "https://github.com/abseil/abseil-cpp/releases/download/20250512.0/\ @@ -28,9 +30,10 @@ tasks: catch2: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DBUILD_TESTING=OFF" @@ -38,40 +41,47 @@ tasks: - "-DCMAKE_CXX_STANDARD=20" - "-DCMAKE_CXX_STANDARD_REQUIRED=ON" - "-DCMAKE_INSTALL_MESSAGE=LAZY" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "Catch2" TARBALL_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" TARBALL_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" date: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "date" TARBALL_SHA256: "7a390f200f0ccd207e8cff6757e04817c1a0aec3e327b006b7eb451c57ee3538" TARBALL_URL: "https://github.com/HowardHinnant/date/archive/refs/tags/v3.0.1.tar.gz" fmt: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DFMT_DOC=OFF" - "-DFMT_TEST=OFF" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "{{.G_FMT_LIB_NAME}}" TARBALL_SHA256: "bc23066d87ab3168f27cef3e97d545fa63314f5c79df5ea444d41d56f962c6af" TARBALL_URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.2.0.tar.gz" liblzma: internal: true + requires: + vars: ["CMAKE_SETTINGS_DIR"] vars: COMMON_CMAKE_GEN_ARGS: - "-DBUILD_TESTING=OFF" @@ -86,11 +96,11 @@ tasks: - "-DXZ_TOOL_XZDEC=OFF" TARBALL_SHA256: "507825b599356c10dca1cd720c9d0d0c9d5400b9de300af00e4d1ea150795543" TARBALL_URL: "https://github.com/tukaani-project/xz/releases/download/v5.8.1/xz-5.8.1.tar.gz" - run: "once" deps: - task: "liblzma-install" vars: BUILD_SHARED_LIBS: true + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" COMMON_CMAKE_GEN_ARGS: ref: ".COMMON_CMAKE_GEN_ARGS" TARBALL_SHA256: "{{.TARBALL_SHA256}}" @@ -98,6 +108,7 @@ tasks: - task: "liblzma-install" vars: BUILD_SHARED_LIBS: false + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" COMMON_CMAKE_GEN_ARGS: ref: ".COMMON_CMAKE_GEN_ARGS" TARBALL_SHA256: "{{.TARBALL_SHA256}}" @@ -106,42 +117,58 @@ tasks: liblzma-install: internal: true requires: - vars: ["BUILD_SHARED_LIBS", "COMMON_CMAKE_GEN_ARGS", "TARBALL_SHA256", "TARBALL_URL"] + vars: + - "BUILD_SHARED_LIBS" + - "CMAKE_SETTINGS_DIR" + - "COMMON_CMAKE_GEN_ARGS" + - "TARBALL_SHA256" + - "TARBALL_URL" cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DBUILD_SHARED_LIBS={{ if .BUILD_SHARED_LIBS }}ON{{ else }}OFF{{ end }}" - >- {{ join " " .COMMON_CMAKE_GEN_ARGS }} + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "LibLZMA-{{ if .BUILD_SHARED_LIBS }}shared{{ else }}static{{ end }}" TARBALL_SHA256: "{{.TARBALL_SHA256}}" TARBALL_URL: "{{.TARBALL_URL}}" log-surgeon: internal: true - run: "once" + vars: + DEPS: + - "fmt" + - "microsoft.gsl" + requires: + vars: ["CMAKE_SETTINGS_DIR"] deps: - - "fmt" - - "microsoft.gsl" + - for: + var: "DEPS" + task: "{{.ITEM}}" + vars: + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_GSL_LIB_NAME}}.cmake" + - "-C {{.CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" + - "-C {{.CMAKE_SETTINGS_DIR}}/{{.G_GSL_LIB_NAME}}.cmake" - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-Dlog_surgeon_BUILD_TESTING=OFF" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "log_surgeon" TARBALL_SHA256: "69a99e0804a52c6b6397c5e7eabecc9bb4915d0145632c66fc63ad13678ff56a" TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/a722d07.tar.gz" lz4: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DBUILD_SHARED_LIBS=ON" @@ -149,6 +176,7 @@ tasks: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DLZ4_BUILD_CLI=OFF" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" CMAKE_SOURCE_DIR: "build/cmake" LIB_NAME: "lz4" TARBALL_SHA256: "537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b" @@ -156,6 +184,8 @@ tasks: mariadb-connector-cpp: internal: true + requires: + vars: ["CMAKE_SETTINGS_DIR"] platforms: ["linux"] preconditions: # NOTE: The MariaDB connector is only required for building Spider, which is only supported on @@ -164,13 +194,13 @@ tasks: # - Make `INSTALL_LAYOUT` configurable. - >- source /etc/os-release && [[ "$ID" == "ubuntu" ]] - run: "once" cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=DEB" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "mariadb-connector-cpp" TARBALL_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" TARBALL_URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/\ @@ -178,25 +208,28 @@ tasks: microsoft.gsl: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DGSL_TEST=OFF" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "{{.G_GSL_LIB_NAME}}" TARBALL_SHA256: "f0e32cb10654fea91ad56bde89170d78cfbf4363ee0b01d8f097de2ba49f6ce9" TARBALL_URL: "https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.tar.gz" mongocxx: internal: true + requires: + vars: ["CMAKE_SETTINGS_DIR"] vars: VERSION: "r4.1.1" - run: "once" cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DBUILD_SHARED_AND_STATIC_LIBS=ON" @@ -204,6 +237,7 @@ tasks: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DENABLE_UNINSTALL=OFF" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "mongocxx" TARBALL_SHA256: "19dff3cf834a3e09229260f22a0325820a7e30c78b294db91794dd934776b33a" TARBALL_URL: "https://github.com/mongodb/mongo-cxx-driver/releases/download/{{.VERSION}}\ @@ -211,21 +245,25 @@ tasks: msgpack-cxx: internal: true + requires: + vars: ["CMAKE_SETTINGS_DIR"] vars: VERSION: "7.0.0" - run: "once" deps: - task: "boost" + vars: + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" + - "-C {{.CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" - "-DMSGPACK_BUILD_DOCS=OFF" - "-DMSGPACK_CXX20=ON" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "msgpack-cxx" TARBALL_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" TARBALL_URL: "https://github.com/msgpack/msgpack-c/releases/download/cpp-{{.VERSION}}\ @@ -233,14 +271,16 @@ tasks: nlohmann_json: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DJSON_BuildTests=OFF" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "nlohmann_json" TARBALL_SHA256: "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406" @@ -251,33 +291,39 @@ tasks: simdjson: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "simdjson" TARBALL_SHA256: "07a1bb3587aac18fd6a10a83fe4ab09f1100ab39f0cb73baea1317826b9f9e0d" TARBALL_URL: "https://github.com/simdjson/simdjson/archive/refs/tags/v3.13.0.tar.gz" spdlog: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] deps: - - "fmt" + - task: "fmt" + vars: + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" + - "-C {{.CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" - "-DSPDLOG_BUILD_EXAMPLE=OFF" - "-DSPDLOG_BUILD_EXAMPLE_HO=OFF" - "-DSPDLOG_FMT_EXTERNAL=ON" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "spdlog" TARBALL_SHA256: "15a04e69c222eb6c01094b5c7ff8a249b36bb22788d72519646fb85feb267e67" @@ -287,53 +333,62 @@ tasks: utfcpp: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DCMAKE_INSTALL_MESSAGE=LAZY" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "utfcpp" TARBALL_SHA256: "6920a6a5d6a04b9a89b2a89af7132f8acefd46e0c2a7b190350539e9213816c0" TARBALL_URL: "https://github.com/nemtrif/utfcpp/archive/refs/tags/v4.0.6.tar.gz" yaml-cpp: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DBUILD_TESTING=OFF" - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "yaml-cpp" TARBALL_SHA256: "43e6a9fcb146ad871515f0d0873947e5d497a1c9c60c58cb102a97b47208b7c3" TARBALL_URL: "https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.tar.gz" ystdlib: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] deps: - task: "boost" + vars: + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" + - "-C {{.CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-Dystdlib_BUILD_TESTING=OFF" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "ystdlib" TARBALL_SHA256: "65990dc2bcc4a355c2181bfe31a7800f492309d1bcd340f52a34e85047e61bc8" TARBALL_URL: "https://github.com/y-scope/ystdlib-cpp/archive/9ed78cd.tar.gz" zlib: internal: true + requires: + vars: ["CMAKE_SETTINGS_DIR"] vars: LIB_NAME: "zlib" - run: "once" cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DBUILD_SHARED_LIBS=ON" @@ -341,15 +396,17 @@ tasks: - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DCMAKE_INSTALL_PREFIX={{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" - "-DZLIB_BUILD_EXAMPLES=OFF" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" LIB_NAME: "{{.LIB_NAME}}" TARBALL_SHA256: "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23" TARBALL_URL: "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz" zstd: internal: true - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - - task: "utils:install-remote-cmake-lib" + - task: "utils:install-remote-cmake-lib-and-generate-settings" vars: CMAKE_GEN_ARGS: - "-DCMAKE_BUILD_TYPE=Release" @@ -359,6 +416,7 @@ tasks: - "-DZSTD_BUILD_SHARED=ON" - "-DZSTD_BUILD_STATIC=ON" - "-DZSTD_BUILD_TESTS=OFF" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" CMAKE_SOURCE_DIR: "build/cmake" LIB_NAME: "zstd" TARBALL_SHA256: "eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" diff --git a/taskfiles/deps/core.yaml b/taskfiles/deps/core.yaml index c3f07c3725..3232dfafb0 100644 --- a/taskfiles/deps/core.yaml +++ b/taskfiles/deps/core.yaml @@ -1,5 +1,10 @@ version: "3" +vars: + # NOTE: This must be kept in-sync with its usage in components/core/CMakeLists.txt + G_DEPS_CPP_CORE_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_DIR}}/cmake-settings/core" + G_DEPS_CPP_CORE_CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CORE_CMAKE_SETTINGS_DIR}}/all-core.cmake" + tasks: default: deps: @@ -12,8 +17,8 @@ tasks: cmds: - task: ":yscope-dev-utils:cmake:install-deps-and-generate-settings" vars: - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" - CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-core.cmake" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CORE_CMAKE_SETTINGS_DIR}}" + CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CORE_CMAKE_SETTINGS_FILE}}" DEP_TASK: "core:core-all-parallel" - task: ":utils:combine-cpp-checksum-files" @@ -48,3 +53,5 @@ tasks: - for: var: "CPP_CORE_TASKS" task: ":{{.ITEM}}" + vars: + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 4ef19cd02c..4e286905a2 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -27,14 +27,11 @@ vars: G_DEPS_CPP_CHECKSUMS_DIR: "{{.G_DEPS_CPP_DIR}}/checksums" G_DEPS_CPP_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/all.md5" - # CMake settings paths - # NOTE: This must be kept in-sync with its usage in components/core/CMakeLists.txt - G_DEPS_CPP_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_DIR}}/cmake-settings" - tasks: default: deps: - "core" + - "spider" - "log-viewer" init: @@ -44,4 +41,3 @@ tasks: - "mkdir -p '{{.G_DEPS_DIR}}'" - "mkdir -p '{{.G_DEPS_CPP_DIR}}'" - "mkdir -p '{{.G_DEPS_CPP_CHECKSUMS_DIR}}'" - - "mkdir -p '{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}'" diff --git a/taskfiles/deps/misc-libs.yaml b/taskfiles/deps/misc-libs.yaml index e09b34b6f4..07c083570c 100644 --- a/taskfiles/deps/misc-libs.yaml +++ b/taskfiles/deps/misc-libs.yaml @@ -42,7 +42,8 @@ tasks: CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" SOURCE_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-src" - run: "once" + requires: + vars: ["CMAKE_SETTINGS_DIR"] deps: - task: "yscope-dev-utils:checksum:validate" vars: @@ -80,7 +81,7 @@ tasks: vars: BUILD_DIR: "{{.BUILD_DIR}}" CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" # This command must be last @@ -89,16 +90,25 @@ tasks: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.INSTALL_PREFIX}}"] + - task: "yscope-dev-utils:cmake:generate-cmake-settings" + vars: + CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" + PACKAGE_INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" + boost: internal: true vars: + LIB_NAME: "Boost" VERSION: "1.87.0" - run: "once" + INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" + requires: + vars: ["CMAKE_SETTINGS_DIR"] cmds: - task: "yscope-dev-utils:boost:download-and-install" vars: - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" FILE_SHA256: "d6c69e4459eb5d6ec208250291221e7ff4a2affde9af6e49c9303b89c687461f" + INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" TARGETS: - "filesystem" - "iostreams" @@ -110,6 +120,11 @@ tasks: URL: "https://github.com/boostorg/boost/releases/download/boost-{{.VERSION}}/\ boost-{{.VERSION}}-b2-nodocs.tar.gz" WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + - task: "yscope-dev-utils:cmake:generate-cmake-settings" + vars: + CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" + PACKAGE_INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" sqlite3: @@ -123,6 +138,10 @@ tasks: INSTALL_SYMLINK: "{{.INSTALL_INCLUDE_DIR}}/{{.LIB_NAME}}" ZIP_FILENAME_STEM: "sqlite-amalgamation-3360000" SRC_DIR: "{{.EXTRACTION_DIR}}/{{.ZIP_FILENAME_STEM}}" + requires: + vars: ["CMAKE_SETTINGS_DIR"] + # TODO: use checksums to properly decide if this task is up-to-date once more than one main deps + # target depends on this library. run: "once" cmds: - task: "yscope-dev-utils:remote:download-and-extract-zip" @@ -136,6 +155,6 @@ tasks: - "ln -s '{{.SRC_DIR}}' '{{.INSTALL_SYMLINK}}'" - |- echo "set(CLP_SQLITE3_SOURCE_DIRECTORY \"{{.SRC_DIR}}\")" \ - > "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" + > "{{.CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" echo "set(CLP_SQLITE3_INCLUDE_DIRECTORY \"{{.INSTALL_INCLUDE_DIR}}\")" \ - >> "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" + >> "{{.CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" diff --git a/taskfiles/deps/spider.yaml b/taskfiles/deps/spider.yaml index ce241fb9e2..21030ebb98 100644 --- a/taskfiles/deps/spider.yaml +++ b/taskfiles/deps/spider.yaml @@ -1,5 +1,9 @@ version: "3" +vars: + G_DEPS_CPP_SPIDER_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_DIR}}/cmake-settings/spider" + G_DEPS_CPP_SPIDER_CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_SPIDER_CMAKE_SETTINGS_DIR}}/all-spider.cmake" + tasks: default: deps: @@ -17,12 +21,13 @@ tasks: - task: ":yscope-dev-utils:cmake:install-remote-tar" vars: CMAKE_PACKAGE_NAME: "spider" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_SPIDER_CMAKE_SETTINGS_DIR}}" TAR_SHA256: "43478b6da99aa2df50c960573ec68c6fcf67cdfbab39b5e314676218aa93b959" TAR_URL: "https://github.com/y-scope/spider/archive/6a7bad3.tar.gz" WORK_DIR: "{{.G_SPIDER_BUILD_DIR}}" CMAKE_JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-spider.cmake" + - "-C {{.G_DEPS_CPP_SPIDER_CMAKE_SETTINGS_FILE}}" - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DSPIDER_BUILD_TESTING=OFF" @@ -34,8 +39,8 @@ tasks: cmds: - task: ":yscope-dev-utils:cmake:install-deps-and-generate-settings" vars: - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" - CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-spider.cmake" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_SPIDER_CMAKE_SETTINGS_DIR}}" + CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_SPIDER_CMAKE_SETTINGS_FILE}}" DEP_TASK: "spider:spider-all-parallel" - task: ":utils:combine-cpp-checksum-files" @@ -57,3 +62,5 @@ tasks: - for: var: "CPP_SPIDER_TASKS" task: ":{{.ITEM}}" + vars: + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" diff --git a/taskfiles/deps/utils.yaml b/taskfiles/deps/utils.yaml index 847f2410d2..9ebc624e5c 100644 --- a/taskfiles/deps/utils.yaml +++ b/taskfiles/deps/utils.yaml @@ -29,6 +29,43 @@ tasks: # Installs a CMake-based library from a remote tarball, generates a corresponding settings file, # and computes a checksum for the installation directory. # + # @param {string} CMAKE_SETTINGS_DIR + # @param {string} LIB_NAME + # @param {string} TARBALL_SHA256 + # @param {string} TARBALL_URL + # @param {string[]} [CMAKE_GEN_ARGS] Any additional arguments to pass to the CMake generate + # command. + # @param {string} [CMAKE_SOURCE_DIR=.] The path, within the tarball extraction directory, + # containing the project's top level CMakeLists.txt. + install-remote-cmake-lib-and-generate-settings: + internal: true + vars: + CMAKE_GEN_ARGS: + ref: "default (list) .CMAKE_GEN_ARGS" + CMAKE_SOURCE_DIR: >- + {{default "." .CMAKE_SOURCE_DIR}} + INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" + INSTALL_DIR_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" + requires: + vars: ["CMAKE_SETTINGS_DIR", "LIB_NAME", "TARBALL_SHA256", "TARBALL_URL"] + cmds: + - task: "install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + ref: ".CMAKE_GEN_ARGS" + LIB_NAME: "{{.LIB_NAME}}" + CMAKE_SOURCE_DIR: "{{.CMAKE_SOURCE_DIR}}" + TARBALL_SHA256: "{{.TARBALL_SHA256}}" + TARBALL_URL: "{{.TARBALL_URL}}" + - task: ":yscope-dev-utils:cmake:generate-cmake-settings" + vars: + CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" + PACKAGE_INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" + + # Installs a CMake-based library from a remote tarball, and computes a checksum for the + # installation directory. + # # @param {string} LIB_NAME # @param {string} TARBALL_SHA256 # @param {string} TARBALL_URL @@ -38,6 +75,7 @@ tasks: # containing the project's top level CMakeLists.txt. install-remote-cmake-lib: internal: true + label: "{{.TASK}}:{{.LIB_NAME}}" vars: CMAKE_GEN_ARGS: ref: "default (list) .CMAKE_GEN_ARGS" @@ -45,6 +83,9 @@ tasks: {{default "." .CMAKE_SOURCE_DIR}} INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" INSTALL_DIR_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" + requires: + vars: ["LIB_NAME", "TARBALL_SHA256", "TARBALL_URL"] + run: "when_changed" deps: - task: ":yscope-dev-utils:checksum:validate" vars: @@ -58,7 +99,6 @@ tasks: ref: ".CMAKE_GEN_ARGS" CMAKE_JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" CMAKE_SOURCE_DIR: "{{.CMAKE_SOURCE_DIR}}" TAR_SHA256: "{{.TARBALL_SHA256}}" TAR_URL: "{{.TARBALL_URL}}" diff --git a/tools/scripts/deps-download/init.sh b/tools/scripts/deps-download/init.sh index f18f1fd1db..a9faa30dd8 100755 --- a/tools/scripts/deps-download/init.sh +++ b/tools/scripts/deps-download/init.sh @@ -10,9 +10,9 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" project_root_dir="$script_dir/../../../" download_dep_script="$script_dir/download-dep.py" -readonly YSCOPE_DEV_UTILS_COMMIT_SHA="f1202b9c21dac05d77632e4d3e1768b11e0dcf47" +readonly YSCOPE_DEV_UTILS_COMMIT_SHA="73d6e964d1432dd5f5d7a9c3d945437d689834cc" python3 "${download_dep_script}" \ - "https://github.com/y-scope/yscope-dev-utils/archive/${YSCOPE_DEV_UTILS_COMMIT_SHA}.zip" \ + "https://github.com/Bill-hbrhbr/yscope-dev-utils/archive/${YSCOPE_DEV_UTILS_COMMIT_SHA}.zip" \ "yscope-dev-utils-${YSCOPE_DEV_UTILS_COMMIT_SHA}" \ "${project_root_dir}/tools/yscope-dev-utils" \ --extract diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index f1202b9c21..73d6e964d1 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit f1202b9c21dac05d77632e4d3e1768b11e0dcf47 +Subproject commit 73d6e964d1432dd5f5d7a9c3d945437d689834cc From 7e27a557735655e850c3875787a93020b55dc5c4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 7 Oct 2025 03:29:35 -0400 Subject: [PATCH 09/13] Artifacts fix --- taskfile.yaml | 2 +- taskfiles/deps/artifacts.yaml | 31 +++++++++++++++++++++++++++++++ taskfiles/deps/main.yaml | 10 ++++++++-- taskfiles/deps/misc-libs.yaml | 31 ------------------------------- 4 files changed, 40 insertions(+), 34 deletions(-) create mode 100644 taskfiles/deps/artifacts.yaml diff --git a/taskfile.yaml b/taskfile.yaml index d2bab9d44d..18b647feb5 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -127,7 +127,7 @@ tasks: - "clp-mcp-server" - "clp-package-utils" - "clp-py-utils" - - "deps:spider:spider-dep" + - "deps:spider" - "init" - "job-orchestration" - "package-venv" diff --git a/taskfiles/deps/artifacts.yaml b/taskfiles/deps/artifacts.yaml new file mode 100644 index 0000000000..1c4167580f --- /dev/null +++ b/taskfiles/deps/artifacts.yaml @@ -0,0 +1,31 @@ +version: "3" + +vars: + G_ANTLR_JAR_FILE: "{{.G_DEPS_CPP_DIR}}/antlr-{{.G_ANTLR_VERSION}}-complete.jar" + +tasks: + antlr-jar: + internal: true + vars: + CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/antlr-jar.md5" + OUTPUT_FILE: "{{.G_ANTLR_JAR_FILE}}" + run: "once" + deps: + - task: "yscope-dev-utils:checksum:validate" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: + - "{{.OUTPUT_FILE}}" + cmds: + - task: "yscope-dev-utils:remote:curl" + vars: + FILE_SHA256: "eae2dfa119a64327444672aff63e9ec35a20180dc5b8090b7a6ab85125df4d76" + OUTPUT_FILE: "{{.OUTPUT_FILE}}" + URL: "https://www.antlr.org/download/antlr-{{.G_ANTLR_VERSION}}-complete.jar" + + # This command must be last + - task: "yscope-dev-utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: + - "{{.OUTPUT_FILE}}" diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 4e286905a2..b6a6645c91 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -1,7 +1,7 @@ version: "3" includes: - # Components + # Main targets core: "core.yaml" spider: "spider.yaml" log-viewer: "log-viewer.yaml" @@ -10,7 +10,10 @@ includes: utils: "utils.yaml" yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" - # Individual library installs + # Individual component tasks + artifacts: + flatten: true + taskfile: "artifacts.yaml" cmake-libs: flatten: true taskfile: "cmake-libs.yaml" @@ -27,6 +30,9 @@ vars: G_DEPS_CPP_CHECKSUMS_DIR: "{{.G_DEPS_CPP_DIR}}/checksums" G_DEPS_CPP_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/all.md5" + # Misc + G_ANTLR_VERSION: "4.13.2" + tasks: default: deps: diff --git a/taskfiles/deps/misc-libs.yaml b/taskfiles/deps/misc-libs.yaml index 07c083570c..d134bffa68 100644 --- a/taskfiles/deps/misc-libs.yaml +++ b/taskfiles/deps/misc-libs.yaml @@ -1,37 +1,6 @@ version: "3" -vars: - # Antlr - G_ANTLR_VERSION: "4.13.2" - G_ANTLR_JAR_FILE: "{{.G_DEPS_CPP_DIR}}/antlr-{{.G_ANTLR_VERSION}}-complete.jar" - tasks: - antlr-jar: - internal: true - vars: - CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/antlr-jar.md5" - OUTPUT_FILE: "{{.G_ANTLR_JAR_FILE}}" - run: "once" - deps: - - task: "yscope-dev-utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: - - "{{.OUTPUT_FILE}}" - cmds: - - task: "yscope-dev-utils:remote:curl" - vars: - FILE_SHA256: "eae2dfa119a64327444672aff63e9ec35a20180dc5b8090b7a6ab85125df4d76" - OUTPUT_FILE: "{{.OUTPUT_FILE}}" - URL: "https://www.antlr.org/download/antlr-{{.G_ANTLR_VERSION}}-complete.jar" - - # This command must be last - - task: "yscope-dev-utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: - - "{{.OUTPUT_FILE}}" - antlr-runtime: internal: true vars: From b43601f4bf08f3873df2b85391d569726ca8981d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 7 Oct 2025 03:35:08 -0400 Subject: [PATCH 10/13] Minor fix --- taskfiles/deps/log-viewer.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/taskfiles/deps/log-viewer.yaml b/taskfiles/deps/log-viewer.yaml index d7c2ac8baa..835195b58b 100644 --- a/taskfiles/deps/log-viewer.yaml +++ b/taskfiles/deps/log-viewer.yaml @@ -5,8 +5,6 @@ vars: tasks: default: - deps: - - task: ":init" cmds: - task: ":yscope-dev-utils:remote:download-and-extract-tar" vars: From 2aa9d34e81de204aafe22e542c02d1f4120c7f6e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 7 Oct 2025 05:00:40 -0400 Subject: [PATCH 11/13] Update dev utils --- tools/scripts/deps-download/init.sh | 2 +- tools/yscope-dev-utils | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/scripts/deps-download/init.sh b/tools/scripts/deps-download/init.sh index a9faa30dd8..3344afa140 100755 --- a/tools/scripts/deps-download/init.sh +++ b/tools/scripts/deps-download/init.sh @@ -10,7 +10,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" project_root_dir="$script_dir/../../../" download_dep_script="$script_dir/download-dep.py" -readonly YSCOPE_DEV_UTILS_COMMIT_SHA="73d6e964d1432dd5f5d7a9c3d945437d689834cc" +readonly YSCOPE_DEV_UTILS_COMMIT_SHA="d7648ea57496ae11540e4bca69f7f2a5eeca6444" python3 "${download_dep_script}" \ "https://github.com/Bill-hbrhbr/yscope-dev-utils/archive/${YSCOPE_DEV_UTILS_COMMIT_SHA}.zip" \ "yscope-dev-utils-${YSCOPE_DEV_UTILS_COMMIT_SHA}" \ diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index 73d6e964d1..d7648ea574 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit 73d6e964d1432dd5f5d7a9c3d945437d689834cc +Subproject commit d7648ea57496ae11540e4bca69f7f2a5eeca6444 From 7d9800299c8efbfd047a0b6f701d46f5792c791b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 7 Oct 2025 05:08:25 -0400 Subject: [PATCH 12/13] Change antlr runtime --- taskfiles/deps/cmake-libs.yaml | 22 ++++++++++++ taskfiles/deps/misc-libs.yaml | 64 ---------------------------------- 2 files changed, 22 insertions(+), 64 deletions(-) diff --git a/taskfiles/deps/cmake-libs.yaml b/taskfiles/deps/cmake-libs.yaml index b4e1bae4ea..998f9c9a19 100644 --- a/taskfiles/deps/cmake-libs.yaml +++ b/taskfiles/deps/cmake-libs.yaml @@ -28,6 +28,28 @@ tasks: TARBALL_URL: "https://github.com/abseil/abseil-cpp/releases/download/20250512.0/\ abseil-cpp-20250512.0.tar.gz" + antlr-runtime: + internal: true + requires: + vars: ["CMAKE_SETTINGS_DIR"] + cmds: + - task: "utils:install-remote-cmake-lib-and-generate-settings" + vars: + CMAKE_GEN_ARGS: + - "-DANTLR4_INSTALL=ON" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + + # Set CMP0135 so that extracted files use the current timestamp as their modification + # timestamp, which ensures the library gets rebuilt if the extracted files change. + - "-DCMAKE_POLICY_DEFAULT_CMP0135=NEW" + CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" + CMAKE_SOURCE_DIR: "runtime/Cpp" + LIB_NAME: "antlr4-runtime" + TARBALL_SHA256: "9f18272a9b32b622835a3365f850dd1063d60f5045fb1e12ce475ae6e18a35bb" + TARBALL_URL: "https://github.com/antlr/antlr4/archive/refs/tags/\ + {{.G_ANTLR_VERSION}}.tar.gz" + catch2: internal: true requires: diff --git a/taskfiles/deps/misc-libs.yaml b/taskfiles/deps/misc-libs.yaml index d134bffa68..ab195354d3 100644 --- a/taskfiles/deps/misc-libs.yaml +++ b/taskfiles/deps/misc-libs.yaml @@ -1,70 +1,6 @@ version: "3" tasks: - antlr-runtime: - internal: true - vars: - LIB_NAME: "antlr4-runtime" - - # Paths - BUILD_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-build" - CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" - INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" - SOURCE_DIR: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-src" - requires: - vars: ["CMAKE_SETTINGS_DIR"] - deps: - - task: "yscope-dev-utils:checksum:validate" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.INSTALL_PREFIX}}"] - - # NOTE: We can't use `yscope-dev-utils:cmake:install-remote-tar` (and by extension - # `utils:install-remote-cmake-lib`) since it doesn't yet support the `INCLUDE_PATTERNS` and - # `NUM_COMPONENTS_TO_STRIP` parameters. - cmds: - - task: "yscope-dev-utils:remote:download-and-extract-tar" - vars: - FILE_SHA256: "9f18272a9b32b622835a3365f850dd1063d60f5045fb1e12ce475ae6e18a35bb" - INCLUDE_PATTERNS: ["*/runtime/Cpp"] - NUM_COMPONENTS_TO_STRIP: 3 - OUTPUT_DIR: "{{.SOURCE_DIR}}" - URL: "https://github.com/antlr/antlr4/archive/refs/tags/{{.G_ANTLR_VERSION}}.tar.gz" - - task: "yscope-dev-utils:cmake:generate" - vars: - BUILD_DIR: "{{.BUILD_DIR}}" - EXTRA_ARGS: - - "-DANTLR4_INSTALL=ON" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - # Set CMP0135 so that extracted files use the current timestamp as their modification - # timestamp, which ensures the library gets rebuilt if the extracted files change. - - "-DCMAKE_POLICY_DEFAULT_CMP0135=NEW" - SOURCE_DIR: "{{.SOURCE_DIR}}" - - task: "yscope-dev-utils:cmake:build" - vars: - BUILD_DIR: "{{.BUILD_DIR}}" - JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" - - task: "yscope-dev-utils:cmake:install" - vars: - BUILD_DIR: "{{.BUILD_DIR}}" - CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" - CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - - # This command must be last - - task: "yscope-dev-utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.INSTALL_PREFIX}}"] - - - task: "yscope-dev-utils:cmake:generate-cmake-settings" - vars: - CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" - CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" - PACKAGE_INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - boost: internal: true vars: From 70a5f78be3f3a72dc98f21949b2d457f04c7d5f2 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 7 Oct 2025 05:19:47 -0400 Subject: [PATCH 13/13] Sync cmake settings file --- components/core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 69987b0ef4..37b332444e 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -123,7 +123,7 @@ if(PROJECT_IS_TOP_LEVEL) # NOTE: We mark the file optional since it's not required if the user happens to have the # dependencies installed already. set(CLP_DEPS_CPP_DIR "${CMAKE_SOURCE_DIR}/../../build/deps/cpp") - include("${CLP_DEPS_CPP_DIR}/cmake-settings/all-core.cmake" + include("${CLP_DEPS_CPP_DIR}/cmake-settings/core/all-core.cmake" OPTIONAL RESULT_VARIABLE CLP_DEPS_CPP_CORE_SETTINGS_FILE_PATH )