fix(windows-miner): back off transient header failures

miners/windows/rustchain_miner_setup.bat

@@ -6,7 +6,7 @@ set "PYTHON_URL=https://www.python.org/ftp/python/3.11.5/python-3.11.5-amd64.exe
 set "PYTHON_INSTALLER=%SCRIPT_DIR%python-3.11.5-amd64.exe"
 set "MINER_URL=https://raw.githubusercontent.com/Scottcjn/Rustchain/main/miners/windows/rustchain_windows_miner.py"
 set "MINER_SCRIPT=%SCRIPT_DIR%rustchain_windows_miner.py"
-set "MINER_SHA256=bd83dd859579750ce5eae54b8cb778af4d82ae3af82b6b5adaf2c0193af1334e"
+set "MINER_SHA256=54dd1fd20111976a1616e01a7dbd87cb2cba312dfe3e8c3b993566583ea4969c"
 set "CRYPTO_URL=https://raw.githubusercontent.com/Scottcjn/Rustchain/main/miners/windows/miner_crypto.py"
 set "CRYPTO_SCRIPT=%SCRIPT_DIR%miner_crypto.py"
 set "CRYPTO_SHA256=ffe2e4c78fdc3f53c129a2ef820cc84549a5720655140e69a3e0baf1f7f385fa"

miners/windows/rustchain_windows_miner.py

@@ -21,6 +21,7 @@
 import uuid
 import subprocess
 import re
+import random
 try:
     import tkinter as tk
     from tkinter import ttk, messagebox, scrolledtext
@@ -82,6 +83,14 @@
 CONFIG_FILE = WALLET_DIR / "config.json"
 WALLET_FILE = WALLET_DIR / "wallet.json"
 
+# Header submissions that fail because the node is temporarily unavailable
+# are retried with bounded exponential backoff.  Client-side validation and
+# authentication failures are terminal for the signed header that produced
+# them and must not be resubmitted unchanged.
+HEADER_RETRY_BASE_SECONDS = 10
+HEADER_RETRY_CAP_SECONDS = 300
+HEADER_RETRYABLE_STATUS_CODES = frozenset((408, 425, 429))
+
 # ---------------------------------------------------------------------------
 # Zephyr dual-mining configuration
 # Zephyr is a privacy coin using the RandomX algorithm (same as Monero).
@@ -149,6 +158,9 @@ def __init__(self, wallet_address):
         self.shares_submitted = 0
         self.shares_accepted = 0
         self._last_submitted_slot = None
+        self._header_retry_slot = None
+        self._header_retry_attempts = 0
+        self._next_header_retry_at = 0.0
         self.miner_id = f"windows_{hashlib.md5(wallet_address.encode()).hexdigest()[:8]}"
         self.node_url = RUSTCHAIN_API
         self.attestation_valid_until = 0
@@ -158,6 +170,8 @@ def __init__(self, wallet_address):
         self.last_entropy = {}
         self.last_attestation_error = ""
         self.last_header_error = ""
+        self.last_header_retryable = False
+        self.header_retry_in_seconds = 0
         # Surfaced fingerprint status — non-empty string means the miner is
         # submitting NO fingerprint and will be enrolled at VM-tier weight
         # (1e-9), i.e. earning ~zero. Shown loudly every attest cycle.
@@ -317,7 +331,7 @@ def _mine_loop(self, callback):
                 if (
                     eligibility.get("eligible")
                     and slot is not None
-                    and slot != self._last_submitted_slot
+                    and self._header_submission_due(slot)
                 ):
                     header = self.generate_header(slot)
                     success = self.submit_header(header)
@@ -340,6 +354,11 @@ def _mine_loop(self, callback):
                         }
                         if not success and self.last_header_error:
                             share_event["error"] = self.last_header_error
+                            share_event["retryable"] = self.last_header_retryable
+                            if self.last_header_retryable:
+                                share_event["retry_in_seconds"] = (
+                                    self.header_retry_in_seconds
+                                )
                         callback(share_event)
                 time.sleep(10)
             except Exception as e:
@@ -738,44 +757,104 @@ def generate_header(self, slot):
         }
 
     def submit_header(self, payload):
-        """Submit one signed header and remember attempted slots.
+        """Submit one signed header and classify failures for retry policy.
 
         Issue #7368: the previous version only updated
         ``_last_submitted_slot`` on success, which meant a rejected or
-        connection-failed header (e.g. HTTP 403 ``no pubkey registered
-        for miner``) was retried every poll for the entire eligibility
-        window. We now record the slot as "handled" regardless of
-        outcome so each slot is attempted at most once. The failure
-        reason is preserved in ``last_header_error`` and surfaced to
-        the headless operator in the share event.
+        connection-failed header was retried every poll for the entire
+        eligibility window. Terminal rejections now mark the slot handled.
+        Network failures and temporary node responses instead schedule a
+        bounded retry; the mining loop rebuilds and signs the header before
+        that retry, so an unchanged rejected payload is never resubmitted.
         """
         slot = payload.get("header", {}).get("slot")
+        self.last_header_retryable = False
+        self.header_retry_in_seconds = 0
         try:
             response = requests.post(
                 f"{self.node_url}/headers/ingest_signed",
                 json=payload,
                 timeout=15,
             )
-            result = response.json()
+            try:
+                result = response.json()
+            except Exception:
+                result = None
             success = (
                 response.status_code == 200
                 and isinstance(result, dict)
                 and bool(result.get("ok"))
             )
             if success:
                 self.last_header_error = ""
+                if slot is not None:
+                    self._last_submitted_slot = slot
+                self._reset_header_retry()
+                return True
             else:
                 self.last_header_error = self._response_diagnostic(response)
+                self.last_header_retryable = self._is_retryable_header_status(
+                    response.status_code
+                )
         except Exception as e:
             self.last_header_error = f"header request failed: {e}"
-            success = False
-        # Mark the slot as handled whether we succeeded or not. The
-        # outer mining loop guards on ``slot != self._last_submitted_slot``,
-        # so this is what stops a 10-second retry storm when a wallet
-        # is unregistered or a node is misconfigured.
-        if slot is not None:
-            self._last_submitted_slot = slot
-        return success
+            self.last_header_retryable = True
+
+        if self.last_header_retryable and slot is not None:
+            self.header_retry_in_seconds = self._schedule_header_retry(slot)
+        else:
+            # A malformed header, bad signature, or other terminal response
+            # cannot improve if the same signed header is sent again.
+            if slot is not None:
+                self._last_submitted_slot = slot
+            self._reset_header_retry()
+        return False
+
+    @staticmethod
+    def _is_retryable_header_status(status_code):
+        """Return whether an HTTP response represents a temporary failure."""
+        if status_code in HEADER_RETRYABLE_STATUS_CODES:
+            return True
+        return 500 <= status_code <= 599
+
+    def _header_submission_due(self, slot, now=None):
+        """Return whether *slot* can be submitted under the retry policy."""
+        if slot == self._last_submitted_slot:
+            return False
+        if slot != self._header_retry_slot:
+            return True
+        if now is None:
+            now = time.monotonic()
+        return now >= self._next_header_retry_at
+
+    def _schedule_header_retry(self, slot, now=None):
+        """Schedule and return the bounded exponential delay for *slot*."""
+        if slot != self._header_retry_slot:
+            self._header_retry_slot = slot
+            self._header_retry_attempts = 0
+        self._header_retry_attempts += 1
+        exponent = min(self._header_retry_attempts - 1, 10)
+        base_delay = min(
+            HEADER_RETRY_BASE_SECONDS * (2 ** exponent),
+            HEADER_RETRY_CAP_SECONDS,
+        )
+        # Spread miners across the retry window after a shared node outage.
+        # Preserve the hard upper bound even once exponential backoff reaches
+        # the cap.
+        delay = min(
+            base_delay * random.uniform(0.75, 1.25),
+            HEADER_RETRY_CAP_SECONDS,
+        )
+        if now is None:
+            now = time.monotonic()
+        self._next_header_retry_at = now + delay
+        return delay
+
+    def _reset_header_retry(self):
+        """Clear retry state after success or a terminal rejection."""
+        self._header_retry_slot = None
+        self._header_retry_attempts = 0
+        self._next_header_retry_at = 0.0
 
 
 # ---------------------------------------------------------------------------
@@ -881,6 +960,10 @@ def _format_headless_event(evt):
         # without attaching a debugger.
         if not evt.get("success") and evt.get("error"):
             line += f" error={evt['error']}"
+            if evt.get("retryable"):
+                line += f" class=retryable retry_in={evt.get('retry_in_seconds')}s"
+            else:
+                line += " class=terminal no_retry"
         return line
     if t == "attest":
         return (

setup_miner.py

@@ -28,7 +28,7 @@
     },
     "Windows": {
         "url": "https://raw.githubusercontent.com/Scottcjn/Rustchain/main/miners/windows/rustchain_windows_miner.py",
-        "sha256": "bd83dd859579750ce5eae54b8cb778af4d82ae3af82b6b5adaf2c0193af1334e",
+        "sha256": "54dd1fd20111976a1616e01a7dbd87cb2cba312dfe3e8c3b993566583ea4969c",
     },
 }
 

tests/test_windows_headless_rejected_slot_retry.py

@@ -10,8 +10,8 @@
 
 These tests prove:
 
-* ``submit_header`` records the slot as handled on both success and
-  failure paths, so a rejected slot is not retried.
+* ``submit_header`` records terminal rejections as handled, while temporary
+  failures remain eligible for bounded exponential-backoff retries.
 * The share event passed to the headless callback carries the safe
   ``last_header_error`` diagnostic when a submission fails.
 * ``_format_headless_event`` includes both the slot and the diagnostic
@@ -64,13 +64,17 @@ def fake_post(*_args, **_kwargs):
     assert success is False
     # The slot is recorded even though the node rejected the header.
     assert miner._last_submitted_slot == 27455
+    assert miner.last_header_retryable is False
+    assert miner._header_submission_due(27455, now=999999.0) is False
     assert "HTTP 403" in miner.last_header_error
     assert "no pubkey registered for miner" in miner.last_header_error
 
 
-def test_submit_header_marks_slot_handled_on_connection_failure(monkeypatch):
+def test_submit_header_backs_off_on_connection_failure(monkeypatch):
     module = _load_windows_miner()
     miner = module.RustChainMiner("RTC02811ff5e2bb4bb4b95eee44c5429cd9525496e7")
+    monkeypatch.setattr(module.time, "monotonic", lambda: 100.0)
+    monkeypatch.setattr(module.random, "uniform", lambda *_args: 1.0)
 
     def fake_post(*_args, **_kwargs):
         raise ConnectionError("connection refused")
@@ -80,11 +84,79 @@ def fake_post(*_args, **_kwargs):
     assert miner._last_submitted_slot is None
     success = miner.submit_header({"header": {"slot": 27456}})
     assert success is False
-    assert miner._last_submitted_slot == 27456
+    assert miner._last_submitted_slot is None
+    assert miner.last_header_retryable is True
+    assert miner.header_retry_in_seconds == 10
+    assert miner._header_submission_due(27456, now=109.9) is False
+    assert miner._header_submission_due(27456, now=110.0) is True
     assert "header request failed" in miner.last_header_error
     assert "connection refused" in miner.last_header_error
 
 
+def test_temporary_http_failures_use_bounded_exponential_backoff(monkeypatch):
+    module = _load_windows_miner()
+    miner = module.RustChainMiner("RTC02811ff5e2bb4bb4b95eee44c5429cd9525496e7")
+    monkeypatch.setattr(module.time, "monotonic", lambda: 200.0)
+    monkeypatch.setattr(module.random, "uniform", lambda *_args: 1.0)
+
+    def fake_post(*_args, **_kwargs):
+        return _StubResponse(503, {"ok": False, "error": "node restarting"})
+
+    monkeypatch.setattr(module.requests, "post", fake_post)
+
+    delays = []
+    for _ in range(7):
+        assert miner.submit_header({"header": {"slot": 27456}}) is False
+        delays.append(miner.header_retry_in_seconds)
+
+    assert delays == [10, 20, 40, 80, 160, 300, 300]
+    assert miner._last_submitted_slot is None
+    assert miner.last_header_retryable is True
+    assert "HTTP 503" in miner.last_header_error
+    assert "node restarting" in miner.last_header_error
+
+
+def test_header_retry_adds_jitter_without_exceeding_cap(monkeypatch):
+    module = _load_windows_miner()
+    miner = module.RustChainMiner("RTC02811ff5e2bb4bb4b95eee44c5429cd9525496e7")
+
+    monkeypatch.setattr(module.random, "uniform", lambda low, high: high)
+
+    assert miner._schedule_header_retry(27464, now=100.0) == 12.5
+    assert miner._next_header_retry_at == 112.5
+
+    # Even the maximum positive jitter remains bounded by the 300 s cap.
+    for _ in range(6):
+        delay = miner._schedule_header_retry(27464, now=100.0)
+    assert delay == module.HEADER_RETRY_CAP_SECONDS
+    assert miner._next_header_retry_at == 400.0
+
+
+def test_plain_text_400_is_terminal_and_surfaces_body(monkeypatch):
+    module = _load_windows_miner()
+    miner = module.RustChainMiner("RTC02811ff5e2bb4bb4b95eee44c5429cd9525496e7")
+
+    class PlainTextResponse:
+        status_code = 400
+        text = "malformed signed header"
+
+        @staticmethod
+        def json():
+            raise ValueError("not JSON")
+
+    monkeypatch.setattr(
+        module.requests,
+        "post",
+        lambda *_args, **_kwargs: PlainTextResponse(),
+    )
+
+    assert miner.submit_header({"header": {"slot": 27463}}) is False
+    assert miner._last_submitted_slot == 27463
+    assert miner.last_header_retryable is False
+    assert "HTTP 400" in miner.last_header_error
+    assert "body=malformed signed header" in miner.last_header_error
+
+
 def test_submit_header_records_slot_on_success(monkeypatch):
     module = _load_windows_miner()
     miner = module.RustChainMiner("RTC02811ff5e2bb4bb4b95eee44c5429cd9525496e7")
@@ -98,6 +170,9 @@ def fake_post(*_args, **_kwargs):
     assert success is True
     assert miner._last_submitted_slot == 27457
     assert miner.last_header_error == ""
+    assert miner.last_header_retryable is False
+    assert miner._header_retry_slot is None
+    assert miner._header_retry_attempts == 0
 
 
 def test_share_event_carries_diagnostic_on_failure(monkeypatch):
@@ -128,10 +203,14 @@ def fake_post(*_args, **_kwargs):
     }
     if not success and miner.last_header_error:
         share_event["error"] = miner.last_header_error
+        share_event["retryable"] = miner.last_header_retryable
+        if miner.last_header_retryable:
+            share_event["retry_in_seconds"] = miner.header_retry_in_seconds
     events.append(share_event)
 
     assert events[0]["success"] is False
     assert events[0]["slot"] == 27458
+    assert events[0]["retryable"] is False
     assert "no pubkey registered for miner" in events[0]["error"]
 
 
@@ -178,6 +257,23 @@ def test_format_headless_event_includes_slot_and_diagnostic():
     assert "slot=27460" in line
     assert "FAIL" in line
     assert "error=HTTP 403 error=no pubkey registered for miner" in line
+    assert "class=terminal no_retry" in line
+
+
+def test_format_headless_event_shows_retry_delay():
+    module = _load_windows_miner()
+    line = module._format_headless_event({
+        "type":             "share",
+        "slot":             27462,
+        "submitted":        7,
+        "accepted":         6,
+        "success":          False,
+        "error":            "HTTP 503 error=node restarting",
+        "retryable":        True,
+        "retry_in_seconds": 40,
+    })
+    assert "class=retryable" in line
+    assert "retry_in=40s" in line
 
 
 def test_format_headless_event_omits_diagnostic_on_success():