diff --git a/sphinx/environment/collectors/toctree.py b/sphinx/environment/collectors/toctree.py
index 5c3d5c97f8c..e35314edd7e 100644
--- a/sphinx/environment/collectors/toctree.py
+++ b/sphinx/environment/collectors/toctree.py
@@ -337,11 +337,26 @@ def register_fignumber(
 
             fignumbers[figure_id] = get_next_fignumber(figtype, secnum)
 
+        def should_process_only_node(node: addnodes.only) -> bool:
+            """Check if content in `only` node should be processed based on tags."""
+            try:
+                return env._tags.eval_condition(node['expr'])
+            except Exception:
+                # If evaluation fails, include the content
+                return True
+
         def _walk_doctree(
             docname: str, doctree: Element, secnum: tuple[int, ...]
         ) -> None:
             nonlocal generated_docnames
             for subnode in doctree.children:
+                # Skip content inside `only` directives that don't match current tags
+                if isinstance(subnode, addnodes.only):
+                    if should_process_only_node(subnode):
+                        # Process the content of the `only` node
+                        _walk_doctree(docname, subnode, secnum)
+                    # Skip this node if it doesn't match tags
+                    continue
                 if isinstance(subnode, nodes.section):
                     next_secnum = get_section_number(docname, subnode)
                     if next_secnum:
diff --git a/tests/roots/test-numfig-only/conf.py b/tests/roots/test-numfig-only/conf.py
new file mode 100644
index 00000000000..c1bc75c7237
--- /dev/null
+++ b/tests/roots/test-numfig-only/conf.py
@@ -0,0 +1,3 @@
+"""Test configuration for numfig with only directive."""
+
+numfig = True
diff --git a/tests/roots/test-numfig-only/index.rst b/tests/roots/test-numfig-only/index.rst
new file mode 100644
index 00000000000..9eec7b4f122
--- /dev/null
+++ b/tests/roots/test-numfig-only/index.rst
@@ -0,0 +1,24 @@
+Test numfig with only directive
+================================
+
+This is a test to reproduce issue #4726.
+
+.. only:: latex
+
+   .. figure:: latex-image.png
+
+      Fig for latex
+
+.. only:: html
+
+   .. figure:: html-image.png
+
+      Fig for html
+
+.. figure:: common-image.png
+
+   Fig for all formats
+
+The figures should be numbered consistently:
+- In HTML: "Fig for html" should be Fig. 1, "Fig for all formats" should be Fig. 2
+- In LaTeX: "Fig for latex" should be Fig. 1, "Fig for all formats" should be Fig. 2
\ No newline at end of file
diff --git a/tests/test_numfig_only.py b/tests/test_numfig_only.py
new file mode 100644
index 00000000000..d83c01608b7
--- /dev/null
+++ b/tests/test_numfig_only.py
@@ -0,0 +1,52 @@
+"""Test numfig with only directive."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from sphinx.testing.util import SphinxTestApp
+
+
+@pytest.mark.sphinx('html', testroot='numfig-only', confoverrides={'numfig': True})
+def test_numfig_with_only_directive_html(app: SphinxTestApp) -> None:
+    """Test that figure numbers are assigned correctly with only directive in HTML."""
+    app.build()
+
+    content = (app.outdir / 'index.html').read_text(encoding='utf-8')
+
+    # In HTML builds, only the HTML figure and common figure should be numbered
+    # The HTML figure should be Fig. 1, and common figure should be Fig. 2
+
+    # Check that the HTML figure is numbered as Fig. 1
+    assert 'Fig. 1' in content
+    # Check that the common figure is numbered as Fig. 2
+    assert 'Fig. 2' in content
+    # Make sure there's no Fig. 3
+    assert 'Fig. 3' not in content
+
+
+@pytest.mark.sphinx('latex', testroot='numfig-only', confoverrides={'numfig': True})
+def test_numfig_with_only_directive_latex(app: SphinxTestApp) -> None:
+    """Test that figure numbers are assigned correctly with only directive in LaTeX.
+
+    Note: This test verifies that figure numbering works correctly with the only directive.
+    The actual content filtering (removing HTML-only content from LaTeX output) is handled
+    by the OnlyNodeTransform which runs later in the build process.
+    """
+    app.build(force_all=True)
+
+    content = (
+        (app.outdir / 'projectnamenotset.tex')
+        .read_text(encoding='utf8')
+        .replace('\r\n', '\n')
+    )
+
+    # Check that the HTML figure is numbered as Fig. 1
+    assert 'Fig. 1' in content
+    # Check that the common figure is numbered as Fig. 2
+    assert 'Fig. 2' in content
+    # Make sure there's no Fig. 3
+    assert 'Fig. 3' not in content