diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5ca020424c0c19..b9130ef6f8dc36 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -120,7 +120,7 @@ jobs: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }} scons-${{ runner.arch }}-macos - name: Building openpilot - run: . .venv/bin/activate && scons -j$(nproc) + run: . .venv/bin/activate && SKIP_PANDA=1 scons -j$(nproc) static_analysis: name: static analysis @@ -157,6 +157,8 @@ jobs: id: setup-step - name: Build openpilot run: ${{ env.RUN }} "scons -j$(nproc)" + env: + SKIP_PANDA: 1 - name: Run unit tests timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} run: | @@ -217,13 +219,7 @@ jobs: simulator_driving: name: simulator driving - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') - || fromJSON('["ubuntu-24.04"]') }} - if: false # FIXME: Started to timeout recently + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: @@ -231,14 +227,26 @@ jobs: - uses: ./.github/workflows/setup-with-retry id: setup-step - name: Build openpilot + timeout-minutes: 45 run: | ${{ env.RUN }} "scons -j$(nproc)" - name: Driving test - timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }} + timeout-minutes: 20 run: | + mkdir -p .ci_artifacts/logs ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ source selfdrive/test/setup_vsound.sh && \ + LOG_ROOT=/tmp/openpilot/.ci_artifacts/logs \ + SIM_LOGS=1 \ + PYOPENCL_CTX=0 \ CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py" + - name: Upload simulator logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: metadrive-sim-logs-${{ github.run_id }}-${{ github.run_attempt }} + path: .ci_artifacts/logs + retention-days: 7 create_raylib_ui_report: name: Create raylib UI Report diff --git a/SConstruct b/SConstruct index 094503cfa7902a..7645b744ea418a 100644 --- a/SConstruct +++ b/SConstruct @@ -195,7 +195,8 @@ Export('messaging') # Build other submodules -SConscript(['panda/SConscript']) +if not os.environ.get("SKIP_PANDA"): + SConscript(['panda/SConscript']) # Build rednose library SConscript(['rednose/SConscript']) diff --git a/tools/mac_setup.sh b/tools/mac_setup.sh index 0ae0b35359e6e8..c4b3eaf5a165e3 100755 --- a/tools/mac_setup.sh +++ b/tools/mac_setup.sh @@ -32,7 +32,11 @@ else brew up fi -brew bundle --file=- <<-EOS +# Clear any cached ARM toolchain pkg that occasionally ships with bad checksum on runners +BREW_CACHE_DIR="$(brew --cache)" +rm -f "${BREW_CACHE_DIR}/downloads"/*arm-gnu-toolchain* || true + +cat > /tmp/op-brew-bundle <<'EOS' brew "git-lfs" brew "capnp" brew "coreutils" @@ -45,11 +49,17 @@ brew "llvm" brew "openssl@3.0" brew "qt@5" brew "zeromq" -cask "gcc-arm-embedded" brew "portaudio" brew "gcc@13" +brew "arm-none-eabi-gcc" EOS +if brew info arm-none-eabi-newlib >/dev/null 2>&1; then + echo 'brew "arm-none-eabi-newlib"' >> /tmp/op-brew-bundle +fi + +brew bundle --file=/tmp/op-brew-bundle + echo "[ ] finished brew install t=$SECONDS" BREW_PREFIX=$(brew --prefix) diff --git a/tools/sim/bridge/metadrive/metadrive_bridge.py b/tools/sim/bridge/metadrive/metadrive_bridge.py index b8dc94cf86fc75..c3f3c6970a8149 100644 --- a/tools/sim/bridge/metadrive/metadrive_bridge.py +++ b/tools/sim/bridge/metadrive/metadrive_bridge.py @@ -48,7 +48,8 @@ def create_map(track_size=60): class MetaDriveBridge(SimulatorBridge): - TICKS_PER_FRAME = 5 + # lower framerate to make CI real-time on standard GitHub runners + TICKS_PER_FRAME = 10 def __init__(self, dual_camera, high_quality, test_duration=math.inf, test_run=False): super().__init__(dual_camera, high_quality) diff --git a/tools/sim/bridge/metadrive/metadrive_process.py b/tools/sim/bridge/metadrive/metadrive_process.py index 2486d87ff997cb..10d4be6ccecebe 100644 --- a/tools/sim/bridge/metadrive/metadrive_process.py +++ b/tools/sim/bridge/metadrive/metadrive_process.py @@ -60,6 +60,7 @@ def metadrive_process(dual_camera: bool, config: dict, camera_array, wide_camera wide_road_image = np.frombuffer(wide_camera_array.get_obj(), dtype=np.uint8).reshape((H, W, 3)) env = MetaDriveEnv(config) + step_every = max(1, int(round(config.get("physics_world_step_size", 0.05) * 100))) def get_current_lane_info(vehicle): _, lane_info, on_lane = vehicle.navigation._get_current_lane(vehicle) @@ -124,7 +125,7 @@ def get_cam_as_rgb(cam): if is_engaged and start_time is None: start_time = time.monotonic() - if rk.frame % 5 == 0: + if rk.frame % step_every == 0: _, _, terminated, _, _ = env.step(vc) timeout = True if start_time is not None and time.monotonic() - start_time >= test_duration else False lane_idx_curr, on_lane = get_current_lane_info(env.vehicle) diff --git a/tools/sim/launch_openpilot.sh b/tools/sim/launch_openpilot.sh index fa5ac731bd3e8f..37c1ce67891012 100755 --- a/tools/sim/launch_openpilot.sh +++ b/tools/sim/launch_openpilot.sh @@ -6,7 +6,12 @@ export SIMULATION="1" export SKIP_FW_QUERY="1" export FINGERPRINT="HONDA_CIVIC_2022" -export BLOCK="${BLOCK},camerad,loggerd,encoderd,micd,logmessaged" +# keep camera process blocked (simulator publishes frames), but optionally allow logging stack +block_list="camerad,micd,logmessaged" +if [[ -z "$SIM_LOGS" ]]; then + block_list="$block_list,loggerd,encoderd" +fi +export BLOCK="${BLOCK},${block_list}" if [[ "$CI" ]]; then # TODO: offscreen UI should work export BLOCK="${BLOCK},ui" diff --git a/tools/sim/tests/test_metadrive_bridge.py b/tools/sim/tests/test_metadrive_bridge.py index 04ce5d584f96b3..fca24777037af0 100644 --- a/tools/sim/tests/test_metadrive_bridge.py +++ b/tools/sim/tests/test_metadrive_bridge.py @@ -12,7 +12,8 @@ class TestMetaDriveBridge(TestSimBridgeBase): @pytest.fixture(autouse=True) def setup_create_bridge(self, test_duration): - self.test_duration = 30 + # match pytest option (default 60s) so wall-clock runtime equals simulated drive time + self.test_duration = test_duration def create_bridge(self): return MetaDriveBridge(False, False, self.test_duration, True) diff --git a/tools/sim/tests/test_sim_bridge.py b/tools/sim/tests/test_sim_bridge.py index d58cddff7fabf0..6102f73b9bbb38 100644 --- a/tools/sim/tests/test_sim_bridge.py +++ b/tools/sim/tests/test_sim_bridge.py @@ -31,7 +31,8 @@ def test_driving(self): p_bridge = bridge.run(q, retries=10) self.processes.append(p_bridge) - max_time_per_step = 60 + # allow slower CI runners without flaking + max_time_per_step = 120 # Wait for bridge to startup start_waiting = time.monotonic() @@ -89,4 +90,12 @@ def teardown_method(self): p.terminate() for p in reversed(self.processes): - p.kill() + if hasattr(p, "join"): + p.join(timeout=10) + if getattr(p, "is_alive", lambda: False)(): + p.kill() + else: # subprocess.Popen + try: + p.wait(timeout=10) + except subprocess.TimeoutExpired: + p.kill()