Merge branch 'develop' into world-map-port-tiles-sequel

Author: chdoc

.github/workflows/build-linux.yml

@@ -66,7 +66,7 @@ on:
         default: false
       gcc-ver:
         type: string
-        default: "10"
+        default: "11"
 
 jobs:
   build-linux64:

.github/workflows/build-windows.yml

@@ -66,12 +66,20 @@ on:
 jobs:
   build-win64:
     name: Build win64
-    runs-on: ubuntu-22.04
+    runs-on: windows-2022
     steps:
-      - name: Install dependencies
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.x'
+      - name: Install build dependencies
+        run: |
+          choco install sccache
+          pip install Jinja2
+      - name: Install doc dependencies
+        if: inputs.docs
         run: |
-          sudo apt-get update
-          sudo apt-get install ccache
+          pip install sphinx
       - name: Clone DFHack
         uses: actions/checkout@v4
         with:
@@ -108,49 +116,82 @@ jobs:
           ref: main
           ssh-key: ${{ secrets.DFHACK_3RDPARTY_TOKEN }}
           path: depends/steam
+      - name: Prepare output directories
+        run: |
+          mkdir output
+          mkdir pdb
+      - name: Get sccache path
+        run: echo ("SCCACHE_DIR=" + $env:LOCALAPPDATA + "\Mozilla\sccache\cache") >> $env:GITHUB_ENV
       - name: Fetch ccache
         if: inputs.platform-files
         uses: actions/cache/restore@v4
         with:
-          path: build/win64-cross/ccache
+          path: ${{ env.SCCACHE_DIR }}
           key: win-msvc-${{ inputs.cache-id }}-${{ github.sha }}
           restore-keys: |
             win-msvc-${{ inputs.cache-id }}
             win-msvc
-      - name: Cross-compile
+      - uses: ilammy/msvc-dev-cmd@v1
+      - name: Configure DFHack
+        run: |
+          cmake `
+            -S . `
+            -B build `
+            -GNinja `
+            -DDFHACK_BUILD_ARCH=64 `
+            -DCMAKE_BUILD_TYPE=Release `
+            -DCMAKE_INSTALL_PREFIX=output `
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache `
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache `
+            -DBUILD_PDBS:BOOL=${{ inputs.cache-id == 'release' }} `
+            -DDFHACK_RUN_URL='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' `
+            -DBUILD_LIBRARY=${{ inputs.platform-files }} `
+            -DBUILD_PLUGINS:BOOL=${{ inputs.platform-files && inputs.plugins }} `
+            -DBUILD_STONESENSE:BOOL=${{ inputs.stonesense }} `
+            -DBUILD_DOCS:BOOL=${{ inputs.docs }} `
+            -DBUILD_DOCS_NO_HTML:BOOL=${{ !inputs.html }} `
+            -DINSTALL_DATA_FILES:BOOL=${{ inputs.common-files }} `
+            -DBUILD_DFLAUNCH:BOOL=${{ inputs.launchdf }} `
+            -DBUILD_TESTS:BOOL=${{ inputs.tests }} `
+            -DBUILD_XMLDUMP:BOOL=${{ inputs.xml-dump-type-sizes }} `
+            ${{ inputs.xml-dump-type-sizes && '-DINSTALL_XMLDUMP:BOOL=1' || '' }}
+      - name: Build DFHack
         env:
-          CMAKE_EXTRA_ARGS: -DBUILD_PDBS:BOOL=${{ inputs.cache-id == 'release' }} -DDFHACK_RUN_URL='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' -DBUILD_LIBRARY=${{ inputs.platform-files }} -DBUILD_PLUGINS:BOOL=${{ inputs.platform-files && inputs.plugins }} -DBUILD_STONESENSE:BOOL=${{ inputs.stonesense }} -DBUILD_DOCS:BOOL=${{ inputs.docs }} -DBUILD_DOCS_NO_HTML:BOOL=${{ !inputs.html }} -DINSTALL_DATA_FILES:BOOL=${{ inputs.common-files }} -DINSTALL_SCRIPTS:BOOL=${{ inputs.common-files }} -DBUILD_DFLAUNCH:BOOL=${{ inputs.launchdf }} -DBUILD_TESTS:BOOL=${{ inputs.tests }} -DBUILD_XMLDUMP:BOOL=${{ inputs.xml-dump-type-sizes }} ${{ inputs.xml-dump-type-sizes && '-DINSTALL_XMLDUMP:BOOL=1' || '' }}
+          SCCACHE_CACHE_SIZE: 500M
         run: |
-          cd build
-          bash -x build-win64-from-linux.sh
+          ninja install -C build
       - name: Finalize cache
         run: |
           cd build
-          ccache -d win64-cross/ccache --show-stats --verbose
-          ccache -d win64-cross/ccache --max-size ${{ inputs.cache-id == 'release' && '500M' || '150M' }}
-          ccache -d win64-cross/ccache --cleanup
-          ccache -d win64-cross/ccache --max-size ${{ inputs.cache-id == 'release' && '2G' || '500M' }}
-          ccache -d win64-cross/ccache --zero-stats
+          sccache --show-stats
+          sccache --zero-stats
       - name: Save ccache
         if: inputs.platform-files && !inputs.cache-readonly
         uses: actions/cache/save@v4
         with:
-          path: build/win64-cross/ccache
+          path: ${{ env.SCCACHE_DIR }}
           key: win-msvc-${{ inputs.cache-id }}-${{ github.sha }}
       - name: Format artifact name
         if: inputs.artifact-name
         id: artifactname
         run: |
-          if test "false" = "${{ inputs.append-date-and-hash }}"; then
-            echo name=${{ inputs.artifact-name }} >> $GITHUB_OUTPUT
-          else
-            echo name=${{ inputs.artifact-name }}-$(date +%Y%m%d)-$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT
-          fi
+          if ("${{ inputs.append-date-and-hash }}" -eq "false") {
+            "name=${{ inputs.artifact-name }}" | Out-File -Append $env:GITHUB_OUTPUT
+          } else {
+            $date = Get-Date -Format "yyyMMdd"
+            $hash = git rev-parse --short HEAD
+            "name=${{ inputs.artifact-name}}-$date-$hash" | Out-File -Append $env:GITHUB_OUTPUT
+          }
+      - name: Prep pdbs
+        if: inputs.artifact-name && inputs.cache-id == 'release'
+        run: |
+          Get-ChildItem -Recurse -File -Path "build" -Filter *.pdb |
+            Copy-Item -Destination "pdb"
       - name: Prep artifact
-        if: inputs.artifact-name
         run: |
-          cd build/win64-cross/output
-          tar cjf ../../../${{ steps.artifactname.outputs.name }}.tar.bz2 .
+          cd output
+          7z a -ttar -so -an . |
+            7z a -si -tbzip2 ../${{ steps.artifactname.outputs.name }}.tar.bz2
       - name: Upload artifact
         if: inputs.artifact-name
         uses: actions/upload-artifact@v4
@@ -162,4 +203,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.artifactname.outputs.name }}_pdb
-          path: build/win64-cross/pdb
+          path: pdb

.github/workflows/generate-symbols.yml

@@ -36,6 +36,7 @@ on:
         default: default
         options:
         - default
+        - experimental
         - testing
         - adventure_test
         - beta
@@ -133,7 +134,7 @@ jobs:
           +quit
         tar xjf dfhack-symbols-linux64-build.tar.bz2 -C DF_steam
         xml/symbols_gen_linux.sh ${{ inputs.version == 'auto' && '50.0' || inputs.version }} STEAM DF_steam
-        if [ "${{ inputs.version }}" = "auto" ]; then
+        if [ "${{ inputs.version }}" == "auto" ]; then
           while pgrep dwarfort; do
             echo "waiting for DF to exit"
             sleep 0.5

.github/workflows/test.yml

@@ -59,9 +59,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - gcc: 10
+        - gcc: 11 # baseline compatibility with ubuntu LTS 22.04
           plugins: "default"
-        - gcc: 12
+        - gcc: 12 # highest available in ubuntu 22.04
           plugins: "all"
 
   test-windows:
@@ -96,7 +96,7 @@ jobs:
       dfhack_repo: ${{ inputs.dfhack_repo }}
       dfhack_ref: ${{ inputs.dfhack_ref }}
       os: ubuntu
-      compiler: gcc-10
+      compiler: gcc-11
       plugins: default
       config: default
 

.github/workflows/watch-df-release.yml

@@ -15,13 +15,17 @@ jobs:
       fail-fast: false
       matrix:
         # df_steam_branch: which DF Steam branch to watch
-        # platform: leave blank to default to all
+        # platform: for symbols generation; leave blank to default to all
         # structures_ref: leave blank to default to master
         # dfhack_ref: leave blank if no structures update is desired
         # steam_branch: leave blank if no DFHack steam push is desired
         include:
           - df_steam_branch: public
-          - df_steam_branch: beta
+#          - df_steam_branch: beta
+#          - df_steam_branch: experimental
+#            structures_ref: experimental
+#            dfhack_ref: experimental
+#            steam_branch: experimental
     steps:
     - name: Fetch state
       uses: actions/cache/restore@v4
@@ -31,7 +35,8 @@ jobs:
     - name: Compare branch metadata
       uses: nick-fields/retry@v3
       with:
-        timeout_minutes: 2
+        timeout_minutes: 5
+        retry_wait_seconds: 60
         command: |
           blob=$(wget 'https://api.steamcmd.net/v1/info/975370?pretty=1' -O- | \
               awk '/^ *"branches"/,0' | \
@@ -117,7 +122,8 @@ jobs:
     - name: Compare versions
       uses: nick-fields/retry@v3
       with:
-        timeout_minutes: 2
+        timeout_minutes: 5
+        retry_wait_seconds: 60
         command: |
           version=$(wget "${{ matrix.url }}" -qO- | tr '"' '\n' | fgrep 'tar.bz2' | head -n1 | sed -r 's/${{ matrix.prefix }}_([0-9]{2})_([0-9]{2})_linux.tar.bz2/\1.\2/')
           echo "latest ${{ matrix.channel }} version: $version"

.gitignore

@@ -49,6 +49,7 @@ build/compile_commands.json
 build/dfhack_setarch.txt
 build/ImportExecutables.cmake
 build/Testing
+build/_deps
 
 # Python binding binaries
 *.pyc
@@ -64,6 +65,7 @@ build/CPack*Config.cmake
 
 # VSCode files
 .vscode
+*.code-workspace
 
 # ctags file
 tags

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ ci:
 repos:
 # shared across repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
+  rev: v6.0.0
   hooks:
     - id: check-added-large-files
     - id: check-case-conflict
@@ -20,11 +20,11 @@ repos:
       args: ['--fix=lf']
     - id: trailing-whitespace
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.30.0
+  rev: 0.37.4
   hooks:
     - id: check-github-workflows
 - repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: v1.5.5
+  rev: v1.5.6
   hooks:
     - id: forbid-tabs
       exclude_types:

CMakeLists.txt

@@ -4,10 +4,11 @@
 cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
 cmake_policy(SET CMP0048 NEW)
 cmake_policy(SET CMP0074 NEW)
+set(CMAKE_INSTALL_MESSAGE "LAZY")
 
 # set up versioning.
-set(DF_VERSION "51.04")
-set(DFHACK_RELEASE "r1")
+set(DF_VERSION "53.15")
+set(DFHACK_RELEASE "r2")
 set(DFHACK_PRERELEASE FALSE)
 
 set(DFHACK_VERSION "${DF_VERSION}-${DFHACK_RELEASE}")
@@ -46,8 +47,8 @@ option(REMOVE_SYMBOLS_FROM_DF_STUBS "Remove debug symbols from DF stubs. (Reduce
 macro(CHECK_GCC compiler_path)
     execute_process(COMMAND ${compiler_path} -dumpversion OUTPUT_VARIABLE GCC_VERSION_OUT)
     string(STRIP "${GCC_VERSION_OUT}" GCC_VERSION_OUT)
-    if(${GCC_VERSION_OUT} VERSION_LESS "10")
-        message(SEND_ERROR "${compiler_path} version ${GCC_VERSION_OUT} cannot be used - use GCC 10 or later")
+    if(${GCC_VERSION_OUT} VERSION_LESS "11")
+        message(SEND_ERROR "${compiler_path} version ${GCC_VERSION_OUT} cannot be used - use GCC 11 or later")
     endif()
 endmacro()
 
@@ -66,7 +67,7 @@ endif()
 
 if(WIN32)
     set(MSVC_MIN_VER 1930)
-    set(MSVC_MAX_VER 1942)
+    set(MSVC_MAX_VER 1944)
     if(NOT MSVC)
         message(SEND_ERROR "No MSVC found! MSVC 2022 version ${MSVC_MIN_VER} to ${MSVC_MAX_VER} is required.")
     elseif((MSVC_VERSION LESS MSVC_MIN_VER) OR (MSVC_VERSION GREATER MSVC_MAX_VER))
@@ -122,6 +123,9 @@ if(MSVC)
     # Enable C5038 - This is equivalent to gcc's -Werror=reorder, which is enabled by default by gcc -Wall
     add_compile_options("/w15038")
 
+    # Enable C4062 - Warns about missing enum case in switch statement, equivalent to gcc -Wswitch
+    add_compile_options("/w14062")
+
     # MSVC panics if an object file contains more than 65,279 sections. this
     # happens quite frequently with code that uses templates, such as vectors.
     add_compile_options("/bigobj")
@@ -223,13 +227,10 @@ set(DFHACK_DATA_DESTINATION hack)
 
 ## where to install things (after the build is done, classic 'make install' or package structure)
 # the dfhack libraries will be installed here:
-if(UNIX)
-    # put the lib into DF/hack
-    set(DFHACK_LIBRARY_DESTINATION ${DFHACK_DATA_DESTINATION})
-else()
-    # windows is crap, therefore we can't do nice things with it. leave the libs on a nasty pile...
-    set(DFHACK_LIBRARY_DESTINATION .)
-endif()
+
+# put the lib into DF/hack
+# windows will find it because dfhooks will `AddDllDirectory` the hack folder at runtime
+set(DFHACK_LIBRARY_DESTINATION ${DFHACK_DATA_DESTINATION})
 
 # external tools will be installed here:
 set(DFHACK_BINARY_DESTINATION .)
@@ -264,7 +265,7 @@ if(UNIX)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32 -march=i686")
     endif()
     string(REPLACE "-DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
-    set(CMAKE_INSTALL_RPATH ${DFHACK_LIBRARY_DESTINATION})
+    set(CMAKE_INSTALL_RPATH "$ORIGIN")
 elseif(MSVC)
     # for msvc, tell it to always use 8-byte pointers to member functions to avoid confusion
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /vmg /vmm /MP")
@@ -344,6 +345,17 @@ if(BUILD_LIBRARY)
     endif()
 endif()
 
+# this can be made conditional once we get to better platform support for std::format
+INCLUDE(FetchContent)
+FetchContent_Declare(
+    fmt
+    GIT_REPOSITORY https://github.com/fmtlib/fmt.git
+    GIT_TAG 790b9389ae99c4ddebdd2736a8602eca1fec684e # 12.1.0 + bugfix for MSVC warning + build time improvements
+)
+FetchContent_MakeAvailable(fmt)
+set(FMTLIB fmt)
+add_definitions("-DUSE_FMTLIB")
+
 if(APPLE)
     # libstdc++ (GCC 4.8.5 for OS X 10.6)
     # fixes crash-on-unwind bug in DF's libstdc++
@@ -425,7 +437,7 @@ macro(dfhack_test name files)
 if(BUILD_LIBRARY AND UNIX AND NOT APPLE) # remove this once our MSVC build env has been updated
     add_executable(${name} ${files})
     target_include_directories(${name} PUBLIC depends/googletest/googletest/include)
-    target_link_libraries(${name} dfhack gtest)
+    target_link_libraries(${name} dfhack ${FMTLIB} gtest)
     add_test(NAME ${name} COMMAND ${name})
 endif()
 endmacro()
@@ -523,7 +535,7 @@ if(BUILD_DOCS)
 
     add_custom_command(OUTPUT ${SPHINX_OUTPUT}
         COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/docs/build.py"
-        ${SPHINX_BUILD_TARGETS} --sphinx="${SPHINX_EXECUTABLE}" -- -q -W
+        ${SPHINX_BUILD_TARGETS} --sphinx="${SPHINX_EXECUTABLE}" --quiet -- -W
         DEPENDS ${SPHINX_DEPS}
         COMMENT "Building documentation with Sphinx"
     )

build/.gitignore

@@ -7,3 +7,5 @@ _CPack_Packages
 *.tar.*
 .cmake
 win64-cross
+dest
+DF