|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Make /etc/hccn.conf a regular file of this container's device NIC IPs. |
| 3 | +
|
| 4 | +Ascend Direct reads address_<phyId>=<ip> from that path. The source |
| 5 | +workflow bind-mounted the host file. This engine does not pass container |
| 6 | +volumes, and the Hong Kong ARC runners ignore Docker --volume, so the |
| 7 | +mount is not here. Recover the IPs already visible in the job: flatten a |
| 8 | +directory-shaped mount, copy a driver copy, or query hccn_tool. Do not |
| 9 | +invent addresses from host NICs. |
| 10 | +
|
| 11 | +The engine then sets ASCEND_RT_VISIBLE_DEVICES from the manifest, which |
| 12 | +is 0,1. That list is not the physical id on every 2-card pod. Run |
| 13 | +38249171972 queried -i 0, got "invalid device id", wrote the one id that |
| 14 | +answered, and the initiator aborted because ADXL asked for phy id 3. |
| 15 | +Probe with that mask unset and write every IP the tool returns. |
| 16 | +Leave ASCEND_RT_VISIBLE_DEVICES alone: the example calls |
| 17 | +aclrtSetDevice with logic ids 0 and 1. Rewriting the mask to physical |
| 18 | +ids 1,2 made that call fail with "Failed to set device ACL" in run |
| 19 | +38375629967. |
| 20 | +""" |
| 21 | + |
| 22 | +from __future__ import annotations |
| 23 | + |
| 24 | +import os |
| 25 | +import re |
| 26 | +import shutil |
| 27 | +import subprocess |
| 28 | + |
| 29 | +HCCN_CONF = '/etc/hccn.conf' |
| 30 | +DRIVER_HCCN_CONF = '/usr/local/Ascend/driver/tools/hccn.conf' |
| 31 | +CARDS_NEEDED = 2 |
| 32 | +PROBE_LIMIT = 16 |
| 33 | + |
| 34 | + |
| 35 | +def _phy_ids_from_npu_smi() -> list[int]: |
| 36 | + info = subprocess.run( |
| 37 | + ['npu-smi', 'info'], |
| 38 | + capture_output=True, |
| 39 | + text=True, |
| 40 | + check=False, |
| 41 | + ) |
| 42 | + found: list[int] = [] |
| 43 | + for line in info.stdout.splitlines(): |
| 44 | + match = re.search(r'\|\s+(\d+)\s+910', line) |
| 45 | + if match is None: |
| 46 | + continue |
| 47 | + phy = int(match.group(1)) |
| 48 | + if phy not in found: |
| 49 | + found.append(phy) |
| 50 | + return found |
| 51 | + |
| 52 | + |
| 53 | +def _parse_hccn_ip(text: str) -> str | None: |
| 54 | + """Parse `hccn_tool -i N -ip -g` output (`ipaddr:1.2.3.4`).""" |
| 55 | + for line in text.splitlines(): |
| 56 | + if 'ipaddr' not in line.lower(): |
| 57 | + continue |
| 58 | + _, _, value = line.partition(':') |
| 59 | + ip = value.strip() |
| 60 | + if ip: |
| 61 | + return ip |
| 62 | + return None |
| 63 | + |
| 64 | + |
| 65 | +def _flatten_dir_shaped_hccn() -> bool: |
| 66 | + """A file mount that ARC created with mkdir -p lands as a directory.""" |
| 67 | + nested = os.path.join(HCCN_CONF, 'hccn.conf') |
| 68 | + if not (os.path.isdir(HCCN_CONF) and os.path.isfile(nested)): |
| 69 | + return False |
| 70 | + with open(nested, 'rb') as handle: |
| 71 | + data = handle.read() |
| 72 | + shutil.rmtree(HCCN_CONF) |
| 73 | + with open(HCCN_CONF, 'wb') as handle: |
| 74 | + handle.write(data) |
| 75 | + print(f'setup: flattened directory-shaped {HCCN_CONF}', flush=True) |
| 76 | + return True |
| 77 | + |
| 78 | + |
| 79 | +def _copy_driver_conf() -> bool: |
| 80 | + if not os.path.isfile(DRIVER_HCCN_CONF): |
| 81 | + return False |
| 82 | + shutil.copyfile(DRIVER_HCCN_CONF, HCCN_CONF) |
| 83 | + print(f'setup: copied {DRIVER_HCCN_CONF} -> {HCCN_CONF}', flush=True) |
| 84 | + return True |
| 85 | + |
| 86 | + |
| 87 | +def _probe_env() -> dict[str, str]: |
| 88 | + env = os.environ.copy() |
| 89 | + env.pop('ASCEND_RT_VISIBLE_DEVICES', None) |
| 90 | + env.pop('ASCEND_VISIBLE_DEVICES', None) |
| 91 | + return env |
| 92 | + |
| 93 | + |
| 94 | +def _query_ip(tool: str, phy: int, env: dict[str, str]) -> str | None: |
| 95 | + query = subprocess.run( |
| 96 | + [tool, '-i', str(phy), '-ip', '-g'], |
| 97 | + capture_output=True, |
| 98 | + text=True, |
| 99 | + check=False, |
| 100 | + env=env, |
| 101 | + ) |
| 102 | + return _parse_hccn_ip(query.stdout) or _parse_hccn_ip(query.stderr) |
| 103 | + |
| 104 | + |
| 105 | +def _write_probed(tool: str) -> None: |
| 106 | + env = _probe_env() |
| 107 | + smi_ids = _phy_ids_from_npu_smi() |
| 108 | + probe_ids: list[int] = [] |
| 109 | + for phy in [*smi_ids, *range(PROBE_LIMIT)]: |
| 110 | + if phy not in probe_ids: |
| 111 | + probe_ids.append(phy) |
| 112 | + resolved: dict[int, str] = {} |
| 113 | + for phy in probe_ids: |
| 114 | + ip = _query_ip(tool, phy, env) |
| 115 | + if ip: |
| 116 | + resolved[phy] = ip |
| 117 | + if len(resolved) < CARDS_NEEDED: |
| 118 | + found = ','.join(str(phy) for phy in sorted(resolved)) or 'none' |
| 119 | + raise SystemExit( |
| 120 | + f'setup: hccn_tool resolved {len(resolved)} device IP(s) ' |
| 121 | + f'[{found}], need {CARDS_NEEDED}') |
| 122 | + with open(HCCN_CONF, 'w', encoding='utf-8') as handle: |
| 123 | + for phy in sorted(resolved): |
| 124 | + handle.write(f'address_{phy}={resolved[phy]}\n') |
| 125 | + ids = ','.join(str(phy) for phy in sorted(resolved)) |
| 126 | + print( |
| 127 | + f'setup: wrote {HCCN_CONF} from hccn_tool ' |
| 128 | + f'({len(resolved)} address lines: {ids})', |
| 129 | + flush=True, |
| 130 | + ) |
| 131 | + |
| 132 | + |
| 133 | +def ensure() -> None: |
| 134 | + if os.path.isfile(HCCN_CONF): |
| 135 | + print(f'setup: {HCCN_CONF} already present', flush=True) |
| 136 | + elif _flatten_dir_shaped_hccn(): |
| 137 | + pass |
| 138 | + elif _copy_driver_conf(): |
| 139 | + pass |
| 140 | + else: |
| 141 | + # setup_example.sh puts driver/tools on PATH before calling this. |
| 142 | + tool = shutil.which('hccn_tool') |
| 143 | + if tool is None: |
| 144 | + raise SystemExit( |
| 145 | + 'setup: no hccn_tool; cannot materialize /etc/hccn.conf') |
| 146 | + _write_probed(tool) |
| 147 | + |
| 148 | + |
| 149 | +if __name__ == '__main__': |
| 150 | + ensure() |
0 commit comments