feat(mooncake): 迁移 example 看护到共用引擎

.github/workflows/mooncake-examples.yml

@@ -0,0 +1,71 @@
+# mooncake examples guard - project thin trigger.
+#
+# Calls the common engine .github/workflows/examples-template.yml;
+# this file only declares what varies per project: schedule,
+# concurrency, upstream repo, dispatch input forwarding, and matrix
+# parallelism. The guard loop (release + workflow-files monitor /
+# manifest-check / run-example matrix / validate-results /
+# save-monitor-state) lives in the engine. See examples-guards/README.md
+# for the design.
+
+name: mooncake-examples
+
+concurrency:
+  # Three groups: schedule runs serialize (a run cancelled mid-way
+  # loses its outcome writeback, and the outcome drives the next
+  # cycle's retry); each PR serializes per number so a new push
+  # cancels the stale run instead of queueing another matrix on the
+  # NPU runners; each manual run stands alone. Group names are
+  # repository-wide, so the PR group carries the workflow name: a bare
+  # pr-<number> would let two projects' runs on one PR cancel each
+  # other.
+  group: ${{ github.event_name == 'schedule' && 'mooncake-examples-schedule' || (github.event_name == 'pull_request' && format('mooncake-examples-pr-{0}', github.event.pull_request.number)) || format('manual-{0}', github.run_id) }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+on:
+  # Daily at 06:00 Beijing time (UTC 22:00). accelerate owns 02:00
+  # Beijing (UTC 18:00); tgi owns 03:00 Beijing (UTC 19:00). The source
+  # repo's mooncake schedule is already commented, so this daily run
+  # does not overlap a live cron there.
+  schedule:
+    - cron: '0 22 * * *'
+  workflow_dispatch:
+    inputs:
+      target_ref:
+        description: Branch, tag, or SHA to test (empty = latest upstream release; falls back to main on API failure)
+        required: false
+        default: ''
+        type: string
+  # PR runs need no merge: GitHub executes the workflow files from the
+  # PR merge ref, so a newly added trigger and engine run on the PR
+  # that adds them. Whether a fork PR waits for a maintainer's approval
+  # first is the repository's fork-PR policy. `pull_request`, not
+  # `pull_request_target`: the declared read permissions are all the
+  # engine needs. paths lists only this project's files, as the
+  # quick-start triggers do: a PR that touches only the shared engine,
+  # scripts or schema starts no project's matrix.
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'examples-guards/mooncake/**'
+      - '.github/workflows/mooncake-examples.yml'
+
+# Engine jobs query the GitHub Job API (validate-results); called
+# workflows run with the caller's GITHUB_TOKEN permissions.
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  mooncake-examples:
+    uses: ./.github/workflows/examples-template.yml
+    with:
+      # Namespaces cache keys (examples-monitor-state-mooncake_*),
+      # artifact names and the scheduling paths under examples-guards/mooncake.
+      project: mooncake
+      upstream_repo: kvcache-ai/Mooncake
+      target_ref: ${{ inputs.target_ref }}
+      # One supported entry, and it needs two 910B cards. Serial is the
+      # whole matrix.
+      max_parallel: 1

examples-guards/mooncake/examples_manifest.yaml

@@ -0,0 +1,90 @@
+version: 1
+scan:
+  root: mooncake-transfer-engine/example
+  include_extensions: ['.cpp', '.py', '.cu']
+supported:
+  - path: mooncake-transfer-engine/example/transfer_engine_ascend_direct_perf.cpp
+    profile: ascend-direct
+    runner: linux-aarch64-a2-2
+    npu_devices: '0,1'
+    image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12
+    exec: build/mooncake-transfer-engine/example/transfer_engine_ascend_direct_perf
+    overlay_args:
+      - --block_iteration=1
+      - --batch_size=2
+      - --block_size=16384
+    timeout_minutes: 120
+unsupported:
+  # v0.3.13.post1 example/CMakeLists.txt:41 builds this only inside
+  # if(USE_ASCEND). transfer_engine_ascend_one_sided.cpp:31 includes
+  # "hccl.h" and :40 defaults protocol to "hccl". This guard passes
+  # -DUSE_ASCEND_DIRECT=ON, which does not define USE_ASCEND.
+  - mooncake-transfer-engine/example/transfer_engine_ascend_one_sided.cpp
+  # Same gate at example/CMakeLists.txt:46. transfer_engine_ascend_perf.cpp:31
+  # includes "hccl.h" and :40 defaults protocol to "hccl".
+  # -DUSE_ASCEND_DIRECT=ON does not build this target.
+  - mooncake-transfer-engine/example/transfer_engine_ascend_perf.cpp
+  # example/CMakeLists.txt:58 builds this only inside
+  # if(USE_ASCEND_HETEROGENEOUS). The initiator at :194 calls
+  # installTransport("ascend"), and :166 puts "cuda:0" in the NIC matrix
+  # beside "npu:0". This runner has no CUDA peer.
+  - mooncake-transfer-engine/example/transfer_engine_heterogeneous_ascend_perf_initiator.cpp
+  # transfer_engine_bench.cpp:91 defaults protocol to "rdma". The help
+  # string is rdma|barex|tcp|efa|nvlink|musa|nvlink_intra|hip|sunrise_link
+  # and does not name ascend or hccl. :95 defaults device_name to
+  # "mlx5_2". The NPU allocator at :136 is under USE_UBSHMEM.
+  - mooncake-transfer-engine/example/transfer_engine_bench.cpp
+  # transfer_engine_bench_with_notify.cpp:77 defaults protocol to "rdma"
+  # (rdma|tcp|nvlink|musa|hip). Under USE_CUDA, :110 calls cudaSetDevice
+  # and :114 calls cudaMalloc.
+  - mooncake-transfer-engine/example/transfer_engine_bench_with_notify.cpp
+  # transfer_engine_bench_with_retry.cpp:62 defaults protocol to "rdma"
+  # (rdma|tcp). Under USE_CUDA, :95 calls cudaSetDevice and :96 calls
+  # cudaMalloc.
+  - mooncake-transfer-engine/example/transfer_engine_bench_with_retry.cpp
+  # transfer_engine_validator.cpp:73 defaults protocol to "rdma"
+  # (rdma|tcp|nvlink|musa|maca|hip). Under USE_CUDA, :112 calls
+  # cudaSetDevice and :116 calls cudaMalloc.
+  - mooncake-transfer-engine/example/transfer_engine_validator.cpp
+  # memory_pool.cpp:81 calls installTransport("rdma"). :89 registers the
+  # buffer with location "cpu:" plus the socket id. No Ascend call.
+  - mooncake-transfer-engine/example/memory_pool.cpp
+  # show_link.cpp:63 calls installTransport("rdma") and prints
+  # engine->showLinks. No device buffer and no Ascend call.
+  - mooncake-transfer-engine/example/show_link.cpp
+  # device_transport_example.cu:27 says "Requires 2 GPUs with P2P access".
+  # :31 includes cuda_runtime.h and :198 calls cudaSetDevice.
+  # example/CMakeLists.txt:82 builds it only inside
+  # if(USE_CUDA AND BUILD_DEVICE_TRANSPORT_EXAMPLE).
+  - mooncake-transfer-engine/example/device_transport_example.cu
+  # nccl_device_transport_example.cu:21 includes cuda_runtime.h, :36
+  # includes nccl_device.cuh, and :227 calls cudaSetDevice.
+  # example/CMakeLists.txt:129 builds it only inside if(USE_NCCL_DEVICE
+  # AND BUILD_NCCL_DEVICE_TRANSPORT_EXAMPLE).
+  - mooncake-transfer-engine/example/nccl_device_transport_example.cu
+  # nccl_host_transport_example.cpp:17 includes nccl.h. :160 logs
+  # "The NCCL host example requires two GPUs". :180 calls
+  # installTransport("nccl") and :182 registers memory as "cuda:" plus
+  # the rank. example/CMakeLists.txt:166 builds it only inside
+  # if(USE_NCCL_HOST AND BUILD_NCCL_HOST_TRANSPORT_EXAMPLE).
+  - mooncake-transfer-engine/example/nccl_host_transport_example.cpp
+  # efa_first_submit_probe.cpp:47 includes efa_transport.h and :86 calls
+  # installTransport("efa"). example/CMakeLists.txt:32 builds it only
+  # inside if(USE_EFA).
+  - mooncake-transfer-engine/example/efa_first_submit_probe.cpp
+  # efa_per_transfer_latency_bench.py:83 starts the bench with
+  # --protocol=efa. :58 builds an ssh argv, ssh_args = ["ssh", ...], and
+  # :173 uses it to start the target on --target_host. Not an on-node
+  # NPU workload.
+  - mooncake-transfer-engine/example/efa_per_transfer_latency_bench.py
+  # batch_register_bench.py:3 titles itself "Batch Memory Registration
+  # Benchmark for Mooncake EFA Transport". :59 defaults --protocol to
+  # "efa".
+  - mooncake-transfer-engine/example/batch_register_bench.py
+  # kvcache_prefix_bench.py:3 titles itself "KV Cache Prefix Transfer
+  # Benchmark for Mooncake EFA Transport". :62 defaults --protocol to
+  # "efa". :166 allocates with torch.zeros(..., device=f"cuda:{gpu_id}").
+  - mooncake-transfer-engine/example/kvcache_prefix_bench.py
+  # bootstrap_server.py:29 routes every method on /metadata into an
+  # in-memory dict via aiohttp. No NPU or transfer call.
+  - mooncake-transfer-engine/example/http-metadata-server-python/bootstrap_server.py

examples-guards/mooncake/scripts/ensure_hccn_conf.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Make /etc/hccn.conf a regular file of this container's device NIC IPs.
+
+Ascend Direct reads address_<phyId>=<ip> from that path. The source
+workflow bind-mounted the host file. This engine does not pass container
+volumes, and the Hong Kong ARC runners ignore Docker --volume, so the
+mount is not here. Recover the IPs already visible in the job: flatten a
+directory-shaped mount, copy a driver copy, or query hccn_tool. Do not
+invent addresses from host NICs.
+
+The engine then sets ASCEND_RT_VISIBLE_DEVICES from the manifest, which
+is 0,1. That list is not the physical id on every 2-card pod. Run
+38249171972 queried -i 0, got "invalid device id", wrote the one id that
+answered, and the initiator aborted because ADXL asked for phy id 3.
+Probe with that mask unset and write every IP the tool returns.
+Leave ASCEND_RT_VISIBLE_DEVICES alone: the example calls
+aclrtSetDevice with logic ids 0 and 1. Rewriting the mask to physical
+ids 1,2 made that call fail with "Failed to set device ACL" in run
+38375629967.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+
+HCCN_CONF = '/etc/hccn.conf'
+DRIVER_HCCN_CONF = '/usr/local/Ascend/driver/tools/hccn.conf'
+CARDS_NEEDED = 2
+PROBE_LIMIT = 16
+
+
+def _phy_ids_from_npu_smi() -> list[int]:
+    info = subprocess.run(
+        ['npu-smi', 'info'],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    found: list[int] = []
+    for line in info.stdout.splitlines():
+        match = re.search(r'\|\s+(\d+)\s+910', line)
+        if match is None:
+            continue
+        phy = int(match.group(1))
+        if phy not in found:
+            found.append(phy)
+    return found
+
+
+def _parse_hccn_ip(text: str) -> str | None:
+    """Parse `hccn_tool -i N -ip -g` output (`ipaddr:1.2.3.4`)."""
+    for line in text.splitlines():
+        if 'ipaddr' not in line.lower():
+            continue
+        _, _, value = line.partition(':')
+        ip = value.strip()
+        if ip:
+            return ip
+    return None
+
+
+def _flatten_dir_shaped_hccn() -> bool:
+    """A file mount that ARC created with mkdir -p lands as a directory."""
+    nested = os.path.join(HCCN_CONF, 'hccn.conf')
+    if not (os.path.isdir(HCCN_CONF) and os.path.isfile(nested)):
+        return False
+    with open(nested, 'rb') as handle:
+        data = handle.read()
+    shutil.rmtree(HCCN_CONF)
+    with open(HCCN_CONF, 'wb') as handle:
+        handle.write(data)
+    print(f'setup: flattened directory-shaped {HCCN_CONF}', flush=True)
+    return True
+
+
+def _copy_driver_conf() -> bool:
+    if not os.path.isfile(DRIVER_HCCN_CONF):
+        return False
+    shutil.copyfile(DRIVER_HCCN_CONF, HCCN_CONF)
+    print(f'setup: copied {DRIVER_HCCN_CONF} -> {HCCN_CONF}', flush=True)
+    return True
+
+
+def _probe_env() -> dict[str, str]:
+    env = os.environ.copy()
+    env.pop('ASCEND_RT_VISIBLE_DEVICES', None)
+    env.pop('ASCEND_VISIBLE_DEVICES', None)
+    return env
+
+
+def _query_ip(tool: str, phy: int, env: dict[str, str]) -> str | None:
+    query = subprocess.run(
+        [tool, '-i', str(phy), '-ip', '-g'],
+        capture_output=True,
+        text=True,
+        check=False,
+        env=env,
+    )
+    return _parse_hccn_ip(query.stdout) or _parse_hccn_ip(query.stderr)
+
+
+def _write_probed(tool: str) -> None:
+    env = _probe_env()
+    smi_ids = _phy_ids_from_npu_smi()
+    probe_ids: list[int] = []
+    for phy in [*smi_ids, *range(PROBE_LIMIT)]:
+        if phy not in probe_ids:
+            probe_ids.append(phy)
+    resolved: dict[int, str] = {}
+    for phy in probe_ids:
+        ip = _query_ip(tool, phy, env)
+        if ip:
+            resolved[phy] = ip
+    if len(resolved) < CARDS_NEEDED:
+        found = ','.join(str(phy) for phy in sorted(resolved)) or 'none'
+        raise SystemExit(
+            f'setup: hccn_tool resolved {len(resolved)} device IP(s) '
+            f'[{found}], need {CARDS_NEEDED}')
+    with open(HCCN_CONF, 'w', encoding='utf-8') as handle:
+        for phy in sorted(resolved):
+            handle.write(f'address_{phy}={resolved[phy]}\n')
+    ids = ','.join(str(phy) for phy in sorted(resolved))
+    print(
+        f'setup: wrote {HCCN_CONF} from hccn_tool '
+        f'({len(resolved)} address lines: {ids})',
+        flush=True,
+    )
+
+
+def ensure() -> None:
+    if os.path.isfile(HCCN_CONF):
+        print(f'setup: {HCCN_CONF} already present', flush=True)
+    elif _flatten_dir_shaped_hccn():
+        pass
+    elif _copy_driver_conf():
+        pass
+    else:
+        # setup_example.sh puts driver/tools on PATH before calling this.
+        tool = shutil.which('hccn_tool')
+        if tool is None:
+            raise SystemExit(
+                'setup: no hccn_tool; cannot materialize /etc/hccn.conf')
+        _write_probed(tool)
+
+
+if __name__ == '__main__':
+    ensure()

examples-guards/mooncake/scripts/expand_overlay_args.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Print OVERLAY_ARGS as one argv token per line.
+
+The manifest stores overlay_args as a JSON array of CLI strings. The
+engine forwards that array in OVERLAY_ARGS. Empty, null, and "" mean
+no extra args. Each string is split with POSIX shell rules so a quoted
+token stays one argv element.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shlex
+
+
+def main() -> None:
+    raw = os.environ.get('OVERLAY_ARGS', '').strip()
+    if not raw or raw in ('null', '""'):
+        return
+    try:
+        items = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise SystemExit(f'OVERLAY_ARGS is not valid JSON: {exc}') from exc
+    if not isinstance(items, list):
+        raise SystemExit(
+            f'OVERLAY_ARGS must be a JSON array, got {type(items).__name__}')
+    for item in items:
+        if not isinstance(item, str):
+            raise SystemExit(
+                'OVERLAY_ARGS items must be strings, '
+                f'got {type(item).__name__}')
+        for token in shlex.split(os.path.expandvars(item), posix=True):
+            if '\n' in token or '\r' in token:
+                raise SystemExit('OVERLAY_ARGS token contains a newline')
+            print(token)
+
+
+if __name__ == '__main__':
+    main()

examples-guards/mooncake/scripts/parse_listen_endpoint.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""Print the last `listening on host:port` line from a target log.
+
+Transfer Engine logs `Transfer Engine RPC using <protocol>, listening on
+<host>:<port>` once the target is ready. The initiator needs that
+endpoint as --segment_id. Exit 1 when the log does not have one yet.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        raise SystemExit(f'usage: {sys.argv[0]} <log>')
+    text = Path(sys.argv[1]).read_text(encoding='utf-8', errors='replace')
+    matches = re.findall(r'listening on (\S+:\d+)', text)
+    if not matches:
+        raise SystemExit(1)
+    print(matches[-1])
+
+
+if __name__ == '__main__':
+    main()