feat: tagfiles generation for cross-referencing in doxygen format

Author: fpelliccioni

.github/workflows/ci.yml

@@ -77,7 +77,7 @@ jobs:
             libcxx-runtimes: libcxx;libcxxabi{{#if (ine os 'windows') }};libunwind{{/if}}
             libcxx-targets: cxx {{#if (ine os 'windows') }}cxxabi unwind{{/if}} install-cxx {{#if (ine os 'windows') }}install-cxxabi install-unwind{{/if}}
             libcxx-cxxflags: {{#if (ieq os 'windows') }}-D__ORDER_LITTLE_ENDIAN__=1234 -D__ORDER_BIG_ENDIAN__=4321 -D__BYTE_ORDER__=__ORDER_LITTLE_ENDIAN__{{/if}}
-            libcxx-cmake-args: -D LLVM_ENABLE_RUNTIMES="{{ libcxx-runtimes }}" {{#if (ieq os 'windows') }}-D LIBCXXABI_USE_LLVM_UNWINDER=OFF -D LIBCXXABI_ENABLE_SHARED=OFF -D LIBCXXABI_ENABLE_STATIC=ON -D LIBCXX_ENABLE_SHARED=OFF -D LIBCXX_NO_VCRUNTIME=ON{{/if}} {{#if (ieq os 'macos') }}-D CMAKE_OSX_ARCHITECTURES=""{{/if}} 
+            libcxx-cmake-args: -D LLVM_ENABLE_RUNTIMES="{{ libcxx-runtimes }}" {{#if (ieq os 'windows') }}-D LIBCXXABI_USE_LLVM_UNWINDER=OFF -D LIBCXXABI_ENABLE_SHARED=OFF -D LIBCXXABI_ENABLE_STATIC=ON -D LIBCXX_ENABLE_SHARED=OFF -D LIBCXX_NO_VCRUNTIME=ON{{/if}} {{#if (ieq os 'macos') }}-D CMAKE_OSX_ARCHITECTURES=""{{/if}}
             mrdocs-ccflags: {{ ccflags }} {{#if (eq compiler 'gcc') }}-static{{/if}} {{#if asan }}-static-libasan{{/if}} {{#if tsan }}-static-libtsan{{/if}}
             mrdocs-cxxflags: {{ cxxflags }} {{#if (eq compiler 'gcc') }}-static{{/if}} {{#if asan }}-static-libasan{{/if}} {{#if tsan }}-static-libtsan{{/if}}
             mrdocs-package-generators: {{#if (ieq os 'windows') }}7Z ZIP WIX{{else}}TGZ TXZ{{/if}}
@@ -233,19 +233,19 @@ jobs:
           build-type: Release
           extra-args: |
             -D LIBXML2_WITH_PROGRAMS=ON
-            -D LIBXML2_WITH_FTP=OFF 
+            -D LIBXML2_WITH_FTP=OFF
             -D LIBXML2_WITH_HTTP=OFF
-            -D LIBXML2_WITH_ICONV=OFF 
+            -D LIBXML2_WITH_ICONV=OFF
             -D LIBXML2_WITH_LEGACY=OFF
-            -D LIBXML2_WITH_LZMA=OFF 
-            -D LIBXML2_WITH_ZLIB=OFF 
+            -D LIBXML2_WITH_LZMA=OFF
+            -D LIBXML2_WITH_ZLIB=OFF
             -D LIBXML2_WITH_ICU=OFF
-            -D LIBXML2_WITH_TESTS=OFF 
-            -D LIBXML2_WITH_HTML=ON 
+            -D LIBXML2_WITH_TESTS=OFF
+            -D LIBXML2_WITH_HTML=ON
             -D LIBXML2_WITH_C14N=ON
-            -D LIBXML2_WITH_CATALOG=ON 
-            -D LIBXML2_WITH_DEBUG=ON 
-            -D LIBXML2_WITH_ISO8859X=ON 
+            -D LIBXML2_WITH_CATALOG=ON
+            -D LIBXML2_WITH_DEBUG=ON
+            -D LIBXML2_WITH_ISO8859X=ON
             -D LIBXML2_WITH_MEM_DEBUG=OFF
             -D LIBXML2_WITH_MODULES=ON
             -D LIBXML2_WITH_OUTPUT=ON
@@ -400,7 +400,7 @@ jobs:
         run: |
           # Find python
           python=$(command -v python3 || command -v python)
-          # The schema in this branch is up to date 
+          # The schema in this branch is up to date
           "$python" ./util/generate-yaml-schema.py --check
           # The schema in the docs folder is valid
           npx -y -p ajv-cli -- ajv compile -s docs/mrdocs.schema.json
@@ -428,12 +428,12 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         run: |
           set -x
-          
+
           if [ -z "$CODECOV_TOKEN" ]; then
               echo "CODECOV_TOKEN is not set. Skipping coverage report."
               exit 0
           fi
-          
+
           # Find gcov
           gcov_tool="gcov"
           for version in "${{steps.setup-cpp.outputs.version-major}}.${{steps.setup-cpp.outputs.version-minor}}" "${{steps.setup-cpp.outputs.version-major}}"; do
@@ -442,18 +442,18 @@ jobs:
               break
             fi
           done
-          
+
           for dir in "./build"; do
               # Generate reports
               echo "Generate report: $dir"
               lcov --rc lcov_branch_coverage=0 --gcov-tool "$gcov_tool" --directory "$dir" --capture --output-file "$dir/all.info"
               lcov --rc lcov_branch_coverage=0 --list "$dir/all.info"
-          
-              # Upload to codecov 
+
+              # Upload to codecov
               echo "Upload to codecov: $dir"
               bash <(curl -s https://codecov.io/bash) -f "$dir/all.info"
           done
-          
+
           # Summary
           echo "# Coverage" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
@@ -516,17 +516,17 @@ jobs:
       - name: Install MrDocs from Package
         run: |
           set -x
-          
+
           # Delete packages/_CPack_Packages files from previous runs
           rm -rf packages/_CPack_Packages
-          
+
           # Print tree structure
           find packages -print | sed 's;[^/]*/;|____;g;s;____|; |;g'
-          
+
           if [[ ${{ runner.os }} != 'Windows' ]]; then
               dest_dir="$HOME/local"
-              mkdir -p "$dest_dir" 
-              find packages -maxdepth 1 -name 'MrDocs-*.tar.gz' -exec tar -vxzf {} -C $dest_dir --strip-components=1 \;              
+              mkdir -p "$dest_dir"
+              find packages -maxdepth 1 -name 'MrDocs-*.tar.gz' -exec tar -vxzf {} -C $dest_dir --strip-components=1 \;
           else
               dest_dir="$GITHUB_WORKSPACE/usr/local"
               dest_dir=$(echo "$dest_dir" | sed 's/\\/\//g')
@@ -541,7 +541,7 @@ jobs:
           fi
           MRDOCS_ROOT="$dest_dir"
           echo -e "MRDOCS_ROOT=$MRDOCS_ROOT" >> $GITHUB_ENV
-          echo -e "$MRDOCS_ROOT/bin" >> $GITHUB_PATH 
+          echo -e "$MRDOCS_ROOT/bin" >> $GITHUB_PATH
           $MRDOCS_ROOT/bin/mrdocs --version
 
       - name: Clone Boost.URL
@@ -571,7 +571,7 @@ jobs:
                   exit 1
               fi
           fi
-          
+
           # Step 3: Ensure llvm-symbolizer is in your PATH
           llvm_bin_path=$(brew --prefix)/opt/llvm/bin
           PATH="$PATH:$llvm_bin_path"
@@ -582,12 +582,12 @@ jobs:
           else
               echo "llvm-symbolizer found at: $LLVM_SYMBOLIZER_PATH"
           fi
-          
+
           # Step 4: Export LLVM_SYMBOLIZER_PATH environment variable
           export LLVM_SYMBOLIZER_PATH="$LLVM_SYMBOLIZER_PATH"
           echo -e "LLVM_SYMBOLIZER_PATH=$LLVM_SYMBOLIZER_PATH" >> $GITHUB_ENV
           echo "Environment variable LLVM_SYMBOLIZER_PATH set to: $LLVM_SYMBOLIZER_PATH"
-          
+
       - name: Generate Landing Page
         working-directory: docs/website
         run: |
@@ -600,7 +600,7 @@ jobs:
       - name: Generate Remote Documentation
         working-directory: docs
         run: |
-          # This playbook renders the documentation 
+          # This playbook renders the documentation
           # content for the website. It includes
           # master, develop, and tags.
           GH_TOKEN="${{ secrets.GITHUB_TOKEN }}"
@@ -620,7 +620,7 @@ jobs:
       - name: Generate Local Documentation
         working-directory: docs
         run: |
-          # This playbook allows us to render the 
+          # This playbook allows us to render the
           # documentation content and visualize it
           # before a workflow that pushes to the
           # website is triggered.
@@ -638,17 +638,18 @@ jobs:
             "xml"
             "html"
           )
-          
+
           # Generate the demos for each variant and generator
           for variant in single multi; do
             for generator in "${generators[@]}"; do
               [[ $generator = xml && $variant = multi ]] && continue
               [[ $variant = multi ]] && multipage="true" || multipage="false"
-              mrdocs --config="$(pwd)/boost/libs/url/doc/mrdocs.yml" "../CMakeLists.txt" --output="$(pwd)/demos/boost-url/$variant/$generator" --multipage=$multipage --generate="$generator"
-              echo "Number of files in demos/boost-url/$variant/$generator: $(find demos/boost-url/$variant/$generator -type f | wc -l)"
+              echo "Generating demos for $variant/$generator/"
+              mrdocs --config="$(pwd)/boost/libs/url/doc/mrdocs.yml" "../CMakeLists.txt" --output="$(pwd)/demos/boost-url/$variant/$generator/" --multipage=$multipage --generate="$generator"
+              echo "Number of files in demos/boost-url/$variant/$generator/: $(find demos/boost-url/$variant/$generator/ -type f | wc -l)"
             done
           done
-          
+
           # Compress demos for the artifact
           tar -cjf $(pwd)/demos.tar.gz -C $(pwd)/demos --strip-components 1 .
           echo "demos_path=$(pwd)/demos.tar.gz" >> $GITHUB_ENV
@@ -757,7 +758,7 @@ jobs:
           # Copy files: This step will copy the landing page and the documentation to www.mrdocs.com
           chmod 755 -R $(pwd)/build/website
           scp -o StrictHostKeyChecking=no -r $(pwd)/build/website/* [email protected]:/var/www/mrdox.com/
-          
+
           # Remove previous demos associated with this tag
           demo_dir="/var/www/mrdox.com/demos/${{ github.ref_name }}"
           ssh -o StrictHostKeyChecking=no [email protected] "rm -rf $demo_dir/boost-url; mkdir -p $demo_dir/boost-url"
@@ -855,10 +856,10 @@ jobs:
         run: |
           # LLVM is be installed with the default compiler
           set -x
-          
+
           # Compress the LLVM installation
           cd ../third-party/llvm-project
-          
+
           # Use 7z on windows
           if [[ ${{ runner.os }} == 'Windows' ]]; then
               7z a -t7z -m0=lzma2 -mx=9 -mfb=64 -md=32m -ms=on ${{ matrix.llvm-archive-filename }} install
@@ -871,15 +872,15 @@ jobs:
         working-directory: ../third-party/llvm-project
         run: |
           set -x
-          
+
           # Ensure required commands exist
           for cmd in ssh-keyscan ssh-agent ssh-add scp; do
             if ! command -v $cmd >/dev/null; then
               echo "$cmd not found"
               exit 1
             fi
           done
-          
+
           # Add SSH key
           mkdir -p ~/.ssh
           ssh-keyscan dev-websites.cpp.al >> ~/.ssh/known_hosts

docs/mrdocs.schema.json

@@ -244,6 +244,12 @@
       "title": "System include paths",
       "type": "array"
     },
+    "tagfile-path": {
+      "default": "<output>/reference.tag.xml",
+      "description": "Specifies the full path (filename) where the generated tagfile should be saved. If left empty, no tagfile will be generated.",
+      "title": "Path for the tagfile",
+      "type": "string"
+    },
     "use-system-stdlib": {
       "default": false,
       "description": "True if the compiler has to use just the system standard library. When set to true, the compiler uses the system standard library instead of the standard library provided by the compiler.",

include/mrdocs/Corpus.hpp

@@ -159,6 +159,39 @@ class MRDOCS_VISIBLE
         }
     }
 
+    /** Visit the members of specified Info.
+     
+        This function iterates the members of the specified
+        Info `I`. For each member associated with a
+        function with the same name as the member, the
+        function object `f` is invoked with the member
+        as the first argument, followed by `args...`.
+
+        When there are more than one member function
+        with the same name, the function object `f` is
+        invoked with an @ref OverloadSet as the first
+        argument, followed by `args...`.    
+        
+        @param I The Info to traverse.
+        @param pred The predicate to use to determine if the member should be visited.
+        @param f The function to invoke with the member as the first argument, followed by `args...`.
+        @param args The arguments to pass to the function.
+    */
+    template <InfoParent T, class Pred, class F, class... Args>
+    void
+    traverseIf(
+        T const& I, Pred&& pred, F&& f, Args&&... args) const
+    {
+        for (auto const& id : I.Members)
+        {
+            if (std::forward<Pred>(pred)(get(id)))
+            {
+                visit(get(id), std::forward<F>(f),
+                      std::forward<Args>(args)...);
+            }
+        }
+    }    
+
     /** Visit the member overloads of specified ScopeInfo.
 
         This function iterates the members of the

include/mrdocs/Generator.hpp

@@ -182,6 +182,29 @@ class MRDOCS_VISIBLE
         Corpus const& corpus) const;
 };
 
+/** Return the full path for single page output.
+
+    This function determines the full path for a single-page output file
+    based on the provided `outputPath` and file `extension`.
+
+    If the `outputPath` already exists:
+    - If it is a directory, appends the default file name with the provided extension.
+    - If it is a file, uses the provided `outputPath` directly.
+
+    If the `outputPath` does not exist:
+    - If it ends with a '/', assumes it is a directory and appends the default file name.
+    - Otherwise, it returns an error because the path is ambiguous.
+
+    @return The full path or an error if the `outputPath` is ambiguous.
+
+    @param outputPath The specified output path, which can be a directory or file.
+    @param extension The file extension to use for single-page output.
+*/
+Expected<std::string>
+getSinglePageFullPath(
+    std::string_view outputPath,
+    std::string_view extension);
+
 } // mrdocs
 } // clang
 

src/lib/Gen/hbs/HandlebarsGenerator.cpp

@@ -15,7 +15,17 @@
 #include "Builder.hpp"
 #include "MultiPageVisitor.hpp"
 #include "SinglePageVisitor.hpp"
+
+#include "lib/Lib/TagfileWriter.hpp"
+#include "lib/Support/RawOstream.hpp"
+
+#include <llvm/ADT/SmallString.h>
+#include <llvm/Support/FileSystem.h>
+#include <llvm/Support/Path.h>
+
 #include <mrdocs/Support/Path.hpp>
+
+#include <fstream>
 #include <sstream>
 
 namespace clang {
@@ -30,6 +40,30 @@ createEscapeFn(HandlebarsGenerator const& gen)
     };
 }
 
+namespace {
+
+/** Generate a Tagfile to enable cross-referencing between documentation symbols. */
+Expected<void>
+generateTagfile(
+    HandlebarsCorpus const& hbsCorpus,
+    std::string_view tagFilePath,
+    std::string_view outputFilesBasePath
+    )
+{
+    MRDOCS_ASSERT(!tagFilePath.empty());
+
+    auto tagFileWriterEx = TagfileWriter::create(hbsCorpus, tagFilePath, outputFilesBasePath);
+    MRDOCS_CHECK_OR(tagFileWriterEx, Unexpected(tagFileWriterEx.error()));
+
+    auto tagfileWriter = std::move(*tagFileWriterEx);
+    tagfileWriter.initialize();
+    visit(hbsCorpus->globalNamespace(), tagfileWriter);
+    tagfileWriter.finalize();
+    return {};
+}
+
+} // anonymous namespace
+
 Expected<ExecutorGroup<Builder>>
 createExecutors(
     HandlebarsGenerator const& gen,
@@ -79,7 +113,17 @@ build(
 {
     if (!corpus.config->multipage)
     {
-        return Generator::build(outputPath, corpus);
+        auto e = Generator::build(outputPath, corpus);
+        if (e.has_value() && !corpus.config->tagfilePath.empty())
+        {
+            // Generate tagfile if specified
+            auto const singlePageFileName = getSinglePageFullPath(outputPath, fileExtension());
+            MRDOCS_CHECK_OR(singlePageFileName, Unexpected(singlePageFileName.error()));
+            HandlebarsCorpus hbsCorpus = createDomCorpus(*this, corpus);
+            return generateTagfile(hbsCorpus, corpus.config->tagfilePath, *singlePageFileName);
+        }
+
+        return e;
     }
 
     // Create corpus and executors
@@ -93,6 +137,12 @@ build(
     // Wait for all executors to finish and check errors
     auto errors = ex.wait();
     MRDOCS_CHECK_OR(errors.empty(), Unexpected(errors));
+
+    if (! corpus.config->tagfilePath.empty())
+    {
+        return generateTagfile(domCorpus, corpus.config->tagfilePath, outputPath);
+    }
+
     return {};
 }
 

src/lib/Gen/xml/XMLTags.cpp

@@ -193,6 +193,9 @@ void
 XMLTags::
 nest(int levels)
 {
+    if (!nesting_)
+        return;
+    
     if(levels > 0)
     {
         indent_.append(levels * 2, ' ');