diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e6754363..20cd0c7fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -227,6 +227,7 @@ jobs: env: successThresholdPercent: 1 xdpmpPollProvider: 'FNDIS' + interfaceCount: 2 # For 'main' commits fullRuntime: 60 # minutes. Update timeout-minutes with any changes. # For PRs @@ -270,7 +271,7 @@ jobs: if: ${{ (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') && matrix.testDriver == 'XDPMP' && matrix.enableTxInspect == 'enableTxInspect' }} shell: PowerShell timeout-minutes: 20 - run: tools/spinxsk.ps1 -Verbose -Stats -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Driver ${{ matrix.testDriver}} -Minutes ${{ env.prRuntime }} -XdpmpPollProvider ${{ env.xdpmpPollProvider }} -SuccessThresholdPercent ${{ env.successThresholdPercent }} -EnableEbpf -TxInspect + run: tools/spinxsk.ps1 -Verbose -Stats -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Driver ${{ matrix.testDriver}} -Minutes ${{ env.prRuntime }} -XdpmpPollProvider ${{ env.xdpmpPollProvider }} -SuccessThresholdPercent ${{ env.successThresholdPercent }} -InterfaceCount ${{ env.interfaceCount }} -EnableEbpf -TxInspect - name: Run spinxsk (PR) if: ${{ (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') && matrix.enableTxInspect == 'disableTxInspect' }} shell: PowerShell @@ -280,7 +281,7 @@ jobs: if: ${{ (github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch') && matrix.testDriver == 'XDPMP' && matrix.enableTxInspect == 'enableTxInspect' }} shell: PowerShell timeout-minutes: 70 - run: tools/spinxsk.ps1 -Verbose -Stats -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Driver ${{ matrix.testDriver}} -Minutes ${{ env.fullRuntime }} -XdpmpPollProvider ${{ env.xdpmpPollProvider }} -SuccessThresholdPercent ${{ env.successThresholdPercent }} -EnableEbpf -TxInspect + run: tools/spinxsk.ps1 -Verbose -Stats -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Driver ${{ matrix.testDriver}} -Minutes ${{ env.fullRuntime }} -XdpmpPollProvider ${{ env.xdpmpPollProvider }} -SuccessThresholdPercent ${{ env.successThresholdPercent }} -InterfaceCount ${{ env.interfaceCount }} -EnableEbpf -TxInspect - name: Run spinxsk (main) if: ${{ github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' && matrix.enableTxInspect == 'disableTxInspect' }} shell: PowerShell diff --git a/submodules/wil b/submodules/wil index 37554797d..2ec8cf0c8 160000 --- a/submodules/wil +++ b/submodules/wil @@ -1 +1 @@ -Subproject commit 37554797d5a1b958a3369836d539220fcb5a31ae +Subproject commit 2ec8cf0c8aee96087897fbe689e926e547bdf68c diff --git a/test/spinxsk/spinxsk.c b/test/spinxsk/spinxsk.c index 6fee77393..58c740490 100644 --- a/test/spinxsk/spinxsk.c +++ b/test/spinxsk/spinxsk.c @@ -48,7 +48,7 @@ #define STRUCT_FIELD_OFFSET(structPtr, field) \ ((UCHAR *)&(structPtr)->field - (UCHAR *)(structPtr)) -#define DEFAULT_IO_BATCH 1 +#define DEFAULT_IO_BATCH 8 #define DEFAULT_DURATION ULONG_MAX #define DEFAULT_QUEUE_COUNT 4 #define DEFAULT_FUZZER_COUNT 3 @@ -83,6 +83,8 @@ CHAR *HELP = " Default: off\n" " -UseFnmp Use FnMp to inject packets in the receive path\n" " Default: off\n" +" -BpfDir Directory containing eBPF programs\n" +" Default: \"\"\n" ; #define ASSERT_FRE(expr) \ @@ -136,8 +138,12 @@ typedef struct { // BOOLEAN isSockRxSet; BOOLEAN isSockTxSet; + BOOLEAN isSockRxFillSet; + BOOLEAN isSockTxCompSet; BOOLEAN isSharedUmemSockRxSet; BOOLEAN isSharedUmemSockTxSet; + BOOLEAN isSharedUmemSockRxFillSet; + BOOLEAN isSharedUmemSockTxCompSet; BOOLEAN isSockBound; BOOLEAN isSockActivated; BOOLEAN isSharedUmemSockBound; @@ -166,7 +172,10 @@ typedef struct { PROGRAM_HANDLE_TYPE Type; union { HANDLE Handle; - struct bpf_object *BpfObject; + struct { + struct bpf_object *BpfObject; + UINT32 EbpfProgramIndex; + }; }; } PROGRAM_HANDLE; @@ -256,6 +265,8 @@ typedef struct { ULONG rxTotal; ULONG txSuccess; ULONG txTotal; + ULONG rxFillSuccess; + ULONG txCompSuccess; ULONG bindSuccess; ULONG bindTotal; ULONG activateSuccess; @@ -264,6 +275,8 @@ typedef struct { ULONG sharedRxTotal; ULONG sharedTxSuccess; ULONG sharedTxTotal; + ULONG sharedRxFillSuccess; + ULONG sharedTxCompSuccess; ULONG sharedBindSuccess; ULONG sharedBindTotal; ULONG sharedActivateSuccess; @@ -296,6 +309,21 @@ UINT32 globalConcurrentWorkerCount = DEFAULT_GLOBAL_CONCURRENT_WORKERS_COUNT; ULONGLONG perfFreq; CONST CHAR *watchdogCmd = ""; CONST CHAR *powershellPrefix; +CONST CHAR *bpfDir = ""; + +typedef struct { + const CHAR *FileName; + LONG InUse; + struct bpf_object *BpfObject; + int ProgramFd; +} EBPF_PROGRAM_ENTRY; + +EBPF_PROGRAM_ENTRY EbpfProgramTable[] = { + { "\\drop.sys", 0, NULL, -1 }, + { "\\pass.sys", 0, NULL, -1 }, + { "\\l1fwd.sys", 0, NULL, -1 }, + { "\\allow_ipv6.sys", 0, NULL, -1 }, +}; ULONG RandUlong( @@ -409,72 +437,145 @@ FuzzHookId( } HRESULT -FuzzProgTestRunXdpEbpfProgram() +TryLoadEbpfProgram( + _In_ UINT32 ProgramIndex, + _Out_ struct bpf_object **OutBpfObject, + _Out_ int *OutProgramFd + ) { - HRESULT Result; + EBPF_PROGRAM_ENTRY *Entry; CHAR Path[MAX_PATH]; - const CHAR *ProgramRelativePath = NULL; - struct bpf_object *BpfObject = NULL; - struct bpf_program *BpfProgram = NULL; - int ProgramFd; + struct bpf_program *BpfProgram; int OriginalThreadPriority; - int PacketInSize = 100; - UCHAR* PacketIn = NULL; - int PacketOutSize = 100; - UCHAR* PacketOut = NULL; - struct bpf_test_run_opts Opts = {0}; - if (!enableEbpf) { - return HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED); - } + ASSERT_FRE(ProgramIndex < RTL_NUMBER_OF(EbpfProgramTable)); + Entry = &EbpfProgramTable[ProgramIndex]; - OriginalThreadPriority = GetThreadPriority(GetCurrentThread()); - ASSERT_FRE(OriginalThreadPriority != THREAD_PRIORITY_ERROR_RETURN); + *OutBpfObject = NULL; + *OutProgramFd = -1; - Result = GetCurrentBinaryPath(Path, sizeof(Path)); - if (FAILED(Result)) { - goto Exit; + // + // Try to acquire exclusive access to this program slot, because otherwise + // eBPF will block waiting for other instances of this program driver to + // unload, resulting in a lack of productive spinning. + // + if (InterlockedCompareExchange(&Entry->InUse, TRUE, FALSE) != 0) { + return HRESULT_FROM_WIN32(ERROR_BUSY); } - ProgramRelativePath = "\\bpf\\allow_ipv6.sys"; - - ASSERT_FRE(strcat_s(Path, sizeof(Path), ProgramRelativePath) == 0); + // + // Build the program path. + // + Path[0] = '\0'; + ASSERT_FRE(strcat_s(Path, sizeof(Path), bpfDir) == 0); + ASSERT_FRE(strcat_s(Path, sizeof(Path), Entry->FileName) == 0); // // To work around control path delays caused by eBPF's epoch implementation, // boost this thread's priority when invoking eBPF APIs. // + OriginalThreadPriority = GetThreadPriority(GetCurrentThread()); + ASSERT_FRE(OriginalThreadPriority != THREAD_PRIORITY_ERROR_RETURN); ASSERT_FRE(SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST)); TraceVerbose("bpf_object__open(%s)", Path); - BpfObject = bpf_object__open(Path); - if (BpfObject == NULL) { + Entry->BpfObject = bpf_object__open(Path); + if (Entry->BpfObject == NULL) { TraceVerbose("bpf_object__open(%s) failed: %d", Path, errno); - Result = E_FAIL; - goto Exit; + goto Fail; } - TraceVerbose("bpf_object__next_program(%p, %p)", BpfObject, NULL); - BpfProgram = bpf_object__next_program(BpfObject, NULL); + TraceVerbose("bpf_object__next_program(%p, %p)", Entry->BpfObject, NULL); + BpfProgram = bpf_object__next_program(Entry->BpfObject, NULL); if (BpfProgram == NULL) { TraceVerbose("bpf_object__next_program failed: %d", errno); - Result = E_FAIL; - goto Exit; + goto Fail; } - TraceVerbose("bpf_object__load(%p)", BpfObject); - if (bpf_object__load(BpfObject) < 0) { + TraceVerbose("bpf_object__load(%p)", Entry->BpfObject); + if (bpf_object__load(Entry->BpfObject) < 0) { TraceVerbose("bpf_object__load failed: %d", errno); - Result = E_FAIL; - goto Exit; + goto Fail; } TraceVerbose("bpf_program__fd(%p)", BpfProgram); - ProgramFd = bpf_program__fd(BpfProgram); - if (ProgramFd < 0) { + Entry->ProgramFd = bpf_program__fd(BpfProgram); + if (Entry->ProgramFd < 0) { TraceVerbose("bpf_program__fd failed: %d", errno); - Result = E_FAIL; - goto Exit; + goto Fail; + } + + *OutBpfObject = Entry->BpfObject; + *OutProgramFd = Entry->ProgramFd; + + ASSERT_FRE(SetThreadPriority(GetCurrentThread(), OriginalThreadPriority)); + return S_OK; + +Fail: + + if (Entry->BpfObject != NULL) { + TraceVerbose("bpf_object__close(%p)", Entry->BpfObject); + bpf_object__close(Entry->BpfObject); + Entry->BpfObject = NULL; + } + Entry->ProgramFd = -1; + InterlockedExchange(&Entry->InUse, FALSE); + + ASSERT_FRE(SetThreadPriority(GetCurrentThread(), OriginalThreadPriority)); + return E_FAIL; +} + +VOID +ReleaseEbpfProgram( + _In_ UINT32 ProgramIndex + ) +{ + EBPF_PROGRAM_ENTRY *Entry; + int OriginalThreadPriority; + + ASSERT_FRE(ProgramIndex < RTL_NUMBER_OF(EbpfProgramTable)); + Entry = &EbpfProgramTable[ProgramIndex]; + ASSERT_FRE(Entry->BpfObject != NULL); + + // + // To work around control path delays caused by eBPF's epoch implementation, + // boost this thread's priority when invoking eBPF APIs. + // + OriginalThreadPriority = GetThreadPriority(GetCurrentThread()); + ASSERT_FRE(OriginalThreadPriority != THREAD_PRIORITY_ERROR_RETURN); + ASSERT_FRE(SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST)); + + TraceVerbose("bpf_object__close(%p) [program %u]", Entry->BpfObject, ProgramIndex); + bpf_object__close(Entry->BpfObject); + Entry->BpfObject = NULL; + Entry->ProgramFd = -1; + + ASSERT_FRE(SetThreadPriority(GetCurrentThread(), OriginalThreadPriority)); + + InterlockedExchange(&Entry->InUse, FALSE); +} + +HRESULT +FuzzProgTestRunXdpEbpfProgram() +{ + HRESULT Result; + struct bpf_object *BpfObject = NULL; + int ProgramFd; + ULONG ProgramIndex; + int PacketInSize = 100; + UCHAR* PacketIn = NULL; + int PacketOutSize = 100; + UCHAR* PacketOut = NULL; + struct bpf_test_run_opts Opts = {0}; + + if (!enableEbpf) { + return HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED); + } + + ProgramIndex = RandUlong() % RTL_NUMBER_OF(EbpfProgramTable); + Result = TryLoadEbpfProgram(ProgramIndex, &BpfObject, &ProgramFd); + if (FAILED(Result)) { + return Result; } switch (RandUlong() % 3) { @@ -531,15 +632,11 @@ FuzzProgTestRunXdpEbpfProgram() // Don't care about the return value here. bpf_prog_test_run_opts(ProgramFd, &Opts); -Exit: + Result = S_OK; - // There is no other use of the loaded program. Unload before exiting. - if (BpfObject != NULL) { - TraceVerbose("bpf_object__close(%p)", BpfObject); - bpf_object__close(BpfObject); - } +Exit: - ASSERT_FRE(SetThreadPriority(GetCurrentThread(), OriginalThreadPriority)); + ReleaseEbpfProgram(ProgramIndex); if (PacketIn != NULL) { free(PacketIn); @@ -559,14 +656,11 @@ AttachXdpEbpfProgram( ) { HRESULT Result; - CHAR Path[MAX_PATH]; - const CHAR *ProgramRelativePath = NULL; struct bpf_object *BpfObject = NULL; - struct bpf_program *BpfProgram = NULL; NET_IFINDEX IfIndex = ifindex; int ProgramFd; int AttachFlags = 0; - int OriginalThreadPriority; + UINT32 ProgramIndex; // // Since eBPF does not support per-queue programs, attach to the entire @@ -583,72 +677,10 @@ AttachXdpEbpfProgram( return HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED); } - OriginalThreadPriority = GetThreadPriority(GetCurrentThread()); - ASSERT_FRE(OriginalThreadPriority != THREAD_PRIORITY_ERROR_RETURN); - - Result = GetCurrentBinaryPath(Path, sizeof(Path)); + ProgramIndex = RandUlong() % RTL_NUMBER_OF(EbpfProgramTable); + Result = TryLoadEbpfProgram(ProgramIndex, &BpfObject, &ProgramFd); if (FAILED(Result)) { - goto Exit; - } - - switch (RandUlong() % 3) { - case 0: - ProgramRelativePath = "\\bpf\\drop.sys"; - break; - case 1: - ProgramRelativePath = "\\bpf\\pass.sys"; - break; - case 2: - ProgramRelativePath = "\\bpf\\l1fwd.sys"; - break; - default: - ASSERT_FRE(FALSE); - } - - ASSERT_FRE(strcat_s(Path, sizeof(Path), ProgramRelativePath) == 0); - - // - // To work around control path delays caused by eBPF's epoch implementation, - // boost this thread's priority when invoking eBPF APIs. - // - ASSERT_FRE(SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST)); - - TraceVerbose("bpf_object__open(%s)", Path); - BpfObject = bpf_object__open(Path); - if (BpfObject == NULL) { - TraceVerbose("bpf_object__open(%s) failed: %d", Path, errno); - Result = E_FAIL; - goto Exit; - } - - TraceVerbose("bpf_object__next_program(%p, %p)", BpfObject, NULL); - BpfProgram = bpf_object__next_program(BpfObject, NULL); - if (BpfProgram == NULL) { - TraceVerbose("bpf_object__next_program failed: %d", errno); - Result = E_FAIL; - goto Exit; - } - - TraceVerbose("bpf_program__set_type(%p, %d)", BpfProgram, BPF_PROG_TYPE_XDP); - if (bpf_program__set_type(BpfProgram, BPF_PROG_TYPE_XDP) < 0) { - TraceVerbose("bpf_program__set_type failed: %d", errno); - Result = E_FAIL; - goto Exit; - } - - TraceVerbose("bpf_object__load(%p)", BpfObject); - if (bpf_object__load(BpfObject) < 0) { - TraceVerbose("bpf_object__load failed: %d", errno); - Result = E_FAIL; - goto Exit; - } - - TraceVerbose("bpf_program__fd(%p)", BpfProgram); - ProgramFd = bpf_program__fd(BpfProgram); - if (ProgramFd < 0) { - TraceVerbose("bpf_program__fd failed: %d", errno); - Result = E_FAIL; - goto Exit; + return Result; } if ((RandUlong() % 2) == 0) { @@ -674,6 +706,7 @@ AttachXdpEbpfProgram( if (RxProgramSet->HandleCount < RTL_NUMBER_OF(RxProgramSet->Handles)) { RxProgramSet->Handles[RxProgramSet->HandleCount].Type = ProgramHandleEbpf; RxProgramSet->Handles[RxProgramSet->HandleCount].BpfObject = BpfObject; + RxProgramSet->Handles[RxProgramSet->HandleCount].EbpfProgramIndex = ProgramIndex; RxProgramSet->HandleCount++; Result = S_OK; } else { @@ -684,14 +717,9 @@ AttachXdpEbpfProgram( Exit: if (FAILED(Result)) { - if (BpfObject != NULL) { - TraceVerbose("bpf_object__close(%p)", BpfObject); - bpf_object__close(BpfObject); - } + ReleaseEbpfProgram(ProgramIndex); } - ASSERT_FRE(SetThreadPriority(GetCurrentThread(), OriginalThreadPriority)); - return Result; } @@ -841,7 +869,8 @@ DetachXdpProgram( ) { HANDLE Handle = NULL; - struct bpf_object *BpfObject = NULL; + UINT32 EbpfProgramIndex = 0; + BOOLEAN DetachEbpf = FALSE; EnterCriticalSection(&RxProgramSet->Lock); if (RxProgramSet->HandleCount > 0) { @@ -853,7 +882,8 @@ DetachXdpProgram( break; case ProgramHandleEbpf: - BpfObject = RxProgramSet->Handles[detachIndex].BpfObject; + EbpfProgramIndex = RxProgramSet->Handles[detachIndex].EbpfProgramIndex; + DetachEbpf = TRUE; break; default: @@ -867,7 +897,7 @@ DetachXdpProgram( ASSERT_FRE(CloseHandle(Handle)); } - if (BpfObject != NULL) { + if (DetachEbpf) { int OriginalThreadPriority = GetThreadPriority(GetCurrentThread()); ASSERT_FRE(OriginalThreadPriority != THREAD_PRIORITY_ERROR_RETURN); @@ -883,10 +913,9 @@ DetachXdpProgram( TraceVerbose("bpf_xdp_detach(%d, 0, NULL)", ifindex); bpf_xdp_detach(ifindex, 0, NULL); - TraceVerbose("bpf_object__close(%p)", BpfObject); - bpf_object__close(BpfObject); - ASSERT_FRE(SetThreadPriority(GetCurrentThread(), OriginalThreadPriority)); + + ReleaseEbpfProgram(EbpfProgramIndex); } } @@ -1595,13 +1624,15 @@ FuzzSocketRxTxSetup( _In_ BOOLEAN RequiresRx, _In_ BOOLEAN RequiresTx, _Inout_ BOOLEAN *WasRxSet, - _Inout_ BOOLEAN *WasTxSet + _Inout_ BOOLEAN *WasTxSet, + _Inout_ BOOLEAN *WasRxFillSet, + _Inout_ BOOLEAN *WasTxCompSet ) { HRESULT res; UINT32 ringSize; - if (RequiresRx) { + if (RequiresRx || !(RandUlong() % 100)) { if (RandUlong() % 2) { FuzzRingSize(Queue, &ringSize); res = @@ -1613,7 +1644,7 @@ FuzzSocketRxTxSetup( } } - if (RequiresTx) { + if (RequiresTx || !(RandUlong() % 100)) { if (RandUlong() % 2) { FuzzRingSize(Queue, &ringSize); res = @@ -1625,18 +1656,28 @@ FuzzSocketRxTxSetup( } } - if (RandUlong() % 2) { - FuzzRingSize(Queue, &ringSize); - res = - XskSetSockopt( - Sock, XSK_SOCKOPT_RX_FILL_RING_SIZE, &ringSize, sizeof(ringSize)); + if (RequiresRx || !(RandUlong() % 100)) { + if (RandUlong() % 2) { + FuzzRingSize(Queue, &ringSize); + res = + XskSetSockopt( + Sock, XSK_SOCKOPT_RX_FILL_RING_SIZE, &ringSize, sizeof(ringSize)); + if (SUCCEEDED(res)) { + WriteBooleanRelease(WasRxFillSet, TRUE); + } + } } - if (RandUlong() % 2) { - FuzzRingSize(Queue, &ringSize); - res = - XskSetSockopt( - Sock, XSK_SOCKOPT_TX_COMPLETION_RING_SIZE, &ringSize, sizeof(ringSize)); + if (RequiresTx || !(RandUlong() % 100)) { + if (RandUlong() % 2) { + FuzzRingSize(Queue, &ringSize); + res = + XskSetSockopt( + Sock, XSK_SOCKOPT_TX_COMPLETION_RING_SIZE, &ringSize, sizeof(ringSize)); + if (SUCCEEDED(res)) { + WriteBooleanRelease(WasTxCompSet, TRUE); + } + } } } @@ -1733,7 +1774,7 @@ FuzzSocketMisc( } if (RandUlong() % 2) { - timeoutMs = RandUlong() % 1000; + timeoutMs = RandUlong() % 3; } if (RandUlong() % 2) { @@ -2563,14 +2604,34 @@ XskDatapathWorkerFn( TraceEnter("q[%u]d[0x%p]", queue->queueId, datapath->threadHandle); if (SUCCEEDED(InitializeDatapath(datapath))) { + UINT32 burstSize = 0; + while (!ReadBooleanNoFence(&done)) { if (ReadNoFence((LONG *)&datapath->shared->state) == ThreadStateReturn) { break; } + // + // Use a random delay to simulate bursts of traffic + // + if (burstSize == 0) { + DWORD status; + UINT32 timeoutMs = RandUlong() % 10; + + status = WaitForSingleObject(stopEvent, timeoutMs); + if (status != WAIT_TIMEOUT) { + break; + } + + burstSize = RandUlong() % 100 + 1; + } + if (!ProcessPkts(datapath)) { break; } + + ASSERT(burstSize > 0); + burstSize--; } if (extraStats) { @@ -2621,14 +2682,16 @@ XskFuzzerWorkerFn( FuzzSocketRxTxSetup( queue, queue->sock, scenarioConfig->sockRx, scenarioConfig->sockTx, - &scenarioConfig->isSockRxSet, &scenarioConfig->isSockTxSet); + &scenarioConfig->isSockRxSet, &scenarioConfig->isSockTxSet, + &scenarioConfig->isSockRxFillSet, &scenarioConfig->isSockTxCompSet); FuzzSocketMisc(queue, queue->sock, &queue->rxProgramSet); if (queue->sharedUmemSock != NULL) { FuzzSocketRxTxSetup( queue, queue->sharedUmemSock, scenarioConfig->sharedUmemSockRx, scenarioConfig->sharedUmemSockTx, - &scenarioConfig->isSharedUmemSockRxSet, &scenarioConfig->isSharedUmemSockTxSet); + &scenarioConfig->isSharedUmemSockRxSet, &scenarioConfig->isSharedUmemSockTxSet, + &scenarioConfig->isSharedUmemSockRxFillSet, &scenarioConfig->isSharedUmemSockTxCompSet); FuzzSocketMisc(queue, queue->sharedUmemSock, &queue->sharedUmemRxProgramSet); } @@ -2653,7 +2716,11 @@ XskFuzzerWorkerFn( queue->queueId, fuzzer->threadHandle); SetEvent(scenarioConfig->completeEvent); } - WaitForSingleObject(stopEvent, 50); + // + // Reduce the rate of fuzzing once the scenario is complete to give threads exercising + // intermediate states priority. + // + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL); } } @@ -2687,12 +2754,18 @@ UpdateSetupStats( if (Queue->scenarioConfig.isSockRxSet) { ++QueueWorker->setupStats.rxSuccess; } + if (Queue->scenarioConfig.isSockRxFillSet) { + ++QueueWorker->setupStats.rxFillSuccess; + } } if (Queue->scenarioConfig.sockTx) { ++QueueWorker->setupStats.txTotal; if (Queue->scenarioConfig.isSockTxSet) { ++QueueWorker->setupStats.txSuccess; } + if (Queue->scenarioConfig.isSockTxCompSet) { + ++QueueWorker->setupStats.txCompSuccess; + } } if (Queue->scenarioConfig.isSockBound) { ++QueueWorker->setupStats.bindSuccess; @@ -2705,12 +2778,18 @@ UpdateSetupStats( if (Queue->scenarioConfig.isSharedUmemSockRxSet) { ++QueueWorker->setupStats.sharedRxSuccess; } + if (Queue->scenarioConfig.isSharedUmemSockRxFillSet) { + ++QueueWorker->setupStats.sharedRxFillSuccess; + } } if (Queue->scenarioConfig.sharedUmemSockTx) { ++QueueWorker->setupStats.sharedTxTotal; if (Queue->scenarioConfig.isSharedUmemSockTxSet) { ++QueueWorker->setupStats.sharedTxSuccess; } + if (Queue->scenarioConfig.isSharedUmemSockTxCompSet) { + ++QueueWorker->setupStats.sharedTxCompSuccess; + } } if (Queue->scenarioConfig.sharedUmemSockRx || Queue->scenarioConfig.sharedUmemSockTx) { ++QueueWorker->setupStats.sharedBindTotal; @@ -2740,21 +2819,29 @@ PrintSetupStats( "\tinit: (%lu / %lu) %lu%%\n" "\tumem: (%lu / %lu) %lu%%\n" "\trx: (%lu / %lu) %lu%%\n" + "\trxFill: (%lu / %lu) %lu%%\n" "\ttx: (%lu / %lu) %lu%%\n" + "\ttxComp: (%lu / %lu) %lu%%\n" "\tbind: (%lu / %lu) %lu%%\n" "\tactivate: (%lu / %lu) %lu%%\n" "\tsharedRx: (%lu / %lu) %lu%%\n" + "\tsharedRxFill: (%lu / %lu) %lu%%\n" "\tsharedTx: (%lu / %lu) %lu%%\n" + "\tsharedTxComp: (%lu / %lu) %lu%%\n" "\tsharedBind: (%lu / %lu) %lu%%\n" "\tsharedActivate: (%lu / %lu) %lu%%\n", setupStats->initSuccess, NumIterations, Pct(setupStats->initSuccess, NumIterations), setupStats->umemSuccess, setupStats->umemTotal, Pct(setupStats->umemSuccess, setupStats->umemTotal), setupStats->rxSuccess, setupStats->rxTotal, Pct(setupStats->rxSuccess, setupStats->rxTotal), + setupStats->rxFillSuccess, setupStats->rxTotal, Pct(setupStats->rxFillSuccess, setupStats->rxTotal), setupStats->txSuccess, setupStats->txTotal, Pct(setupStats->txSuccess, setupStats->txTotal), + setupStats->txCompSuccess, setupStats->txTotal, Pct(setupStats->txCompSuccess, setupStats->txTotal), setupStats->bindSuccess, setupStats->bindTotal, Pct(setupStats->bindSuccess, setupStats->bindTotal), setupStats->activateSuccess, setupStats->activateTotal, Pct(setupStats->activateSuccess, setupStats->activateTotal), setupStats->sharedRxSuccess, setupStats->sharedRxTotal, Pct(setupStats->sharedRxSuccess, setupStats->sharedRxTotal), + setupStats->sharedRxFillSuccess, setupStats->sharedRxTotal, Pct(setupStats->sharedRxFillSuccess, setupStats->sharedRxTotal), setupStats->sharedTxSuccess, setupStats->sharedTxTotal, Pct(setupStats->sharedTxSuccess, setupStats->sharedTxTotal), + setupStats->sharedTxCompSuccess, setupStats->sharedTxTotal, Pct(setupStats->sharedTxCompSuccess, setupStats->sharedTxTotal), setupStats->sharedBindSuccess, setupStats->sharedBindTotal, Pct(setupStats->sharedBindSuccess, setupStats->sharedBindTotal), setupStats->sharedActivateSuccess, setupStats->sharedActivateTotal, Pct(setupStats->sharedActivateSuccess, setupStats->sharedActivateTotal)); } @@ -2819,6 +2906,7 @@ QueueWorkerFn( // Let datapath thread/s pump datapath for set duration. // TraceVerbose("q[%u]: letting datapath pump", queue->queueId); + WaitForSingleObject(stopEvent, 500); // // Signal and wait for datapath thread/s to return. @@ -2983,6 +3071,13 @@ AdminFn( TraceVerbose("admin iter"); if (!cleanDatapath && !(RandUlong() % 10)) { + // + // Restart the network adapter. When fault injection is enabled, + // XDP may not successfully rebind to the adapter after the restart, + // so this function should ideally restart the adapter until XDP + // appears to have a successful binding. Over long enough runtimes, + // the failure case is currently amortized to an acceptable level. + // INT exitCode; TraceVerbose("admin: restart adapter"); RtlZeroMemory(cmdBuff, sizeof(cmdBuff)); @@ -3149,6 +3244,12 @@ ParseArgs( } else if (!strcmp(argv[i], "-UseFnmp")) { useFnmp = TRUE; TraceVerbose("useFnmp=%!BOOLEAN!", useFnmp); + } else if (!strcmp(argv[i], "-BpfDir")) { + if (++i >= argc) { + Usage(); + } + bpfDir = argv[i]; + TraceVerbose("bpfDir=%s", bpfDir); } else { Usage(); } diff --git a/test/xdpmp/inf/xdpmp.inx b/test/xdpmp/inf/xdpmp.inx index 6c2ac620c..e6d37071a 100644 --- a/test/xdpmp/inf/xdpmp.inx +++ b/test/xdpmp/inf/xdpmp.inx @@ -142,6 +142,14 @@ HKR, Ndi\Params\PollProvider\Enum, "0", 0, %POLL_PROVIDER_NDIS% HKR, Ndi\Params\PollProvider\Enum, "1", 0, %POLL_PROVIDER_FNDIS% +; MACLastByte + HKR, Ndi\Params\MACLastByte, ParamDesc, 0, "MACLastByte" + HKR, Ndi\Params\MACLastByte, default, 0, "0" + HKR, Ndi\Params\MACLastByte, type, 0, "int" + HKR, Ndi\Params\MACLastByte, min, 0, "0" + HKR, Ndi\Params\MACLastByte, max, 0, "254" + HKR, Ndi\Params\MACLastByte, step, 0, "1" + HKR, Ndi\Params\MACLastByte, Optional, 0, "0" [SourceDisksNames] ; diskid = description[, [tagfile] [, , subdir]] diff --git a/test/xdpmp/miniport.c b/test/xdpmp/miniport.c index 89553089f..98630f9c0 100644 --- a/test/xdpmp/miniport.c +++ b/test/xdpmp/miniport.c @@ -19,6 +19,7 @@ NDIS_STRING RegRxPattern = NDIS_STRING_CONST("RxPattern"); NDIS_STRING RegRxPatternCopy = NDIS_STRING_CONST("RxPatternCopy"); NDIS_STRING RegPollProvider = NDIS_STRING_CONST("PollProvider"); NDIS_STRING RxRscSegmentCount = NDIS_STRING_CONST("RxRscSegmentCount"); +NDIS_STRING RegMacLastByte = NDIS_STRING_CONST("MACLastByte"); PCSTR MpDriverFriendlyName = "XDPMP"; UCHAR MpMacAddressBase[MAC_ADDR_LEN] = {0x22, 0x22, 0x22, 0x22, 0x00, 0x00}; @@ -49,6 +50,14 @@ UCHAR MpMacAddressBase[MAC_ADDR_LEN] = {0x22, 0x22, 0x22, 0x22, 0x00, 0x00}; #define MAX_GSO_SIZE 62000 +#define DEFAULT_RATE_SIM_INTERVAL 1000 // 1ms + +#if DBG +#define DEFAULT_RATE_SIM_FRAMES_PER_INTERVAL 10 /* 10 Kpps */ +#else +#define DEFAULT_RATE_SIM_FRAMES_PER_INTERVAL 1000 /* 1 Mpps */ +#endif + // // The driver only supports the driver API version in the DDK or higher. // Drivers can set lower values for backwards compatibility. @@ -305,13 +314,13 @@ MpInitialize( goto Exit; } + NdisMoveMemory(Adapter->MACAddress, MpMacAddressBase, MAC_ADDR_LEN); + Status = MpReadConfiguration(Adapter); if (Status != NDIS_STATUS_SUCCESS) { goto Exit; } - NdisMoveMemory(Adapter->MACAddress, MpMacAddressBase, MAC_ADDR_LEN); - Adapter->IfIndex = InitParameters->IfIndex; NdisZeroMemory( @@ -936,6 +945,7 @@ MpReadConfiguration( NDIS_STATUS Status; NDIS_CONFIGURATION_OBJECT ConfigObject; NDIS_CONFIGURATION_PARAMETER *ConfigParam; + ULONG MacLastByte = Adapter->MACAddress[MAC_ADDR_LEN - 1]; ConfigObject.Header.Type = NDIS_OBJECT_TYPE_CONFIGURATION_OBJECT; ConfigObject.Header.Revision = NDIS_CONFIGURATION_OBJECT_REVISION_1; @@ -1039,9 +1049,9 @@ MpReadConfiguration( TRY_READ_INT_CONFIGURATION(ConfigHandle, RegRxPatternCopy, &Adapter->RxPatternCopy); Adapter->RxPatternCopy = !!Adapter->RxPatternCopy; - Adapter->RateSim.IntervalUs = 1000; // 1ms - Adapter->RateSim.RxFramesPerInterval = 1000; // 1Mpps - Adapter->RateSim.TxFramesPerInterval = 1000; // 1Mpps + Adapter->RateSim.IntervalUs = DEFAULT_RATE_SIM_INTERVAL; + Adapter->RateSim.RxFramesPerInterval = DEFAULT_RATE_SIM_FRAMES_PER_INTERVAL; + Adapter->RateSim.TxFramesPerInterval = DEFAULT_RATE_SIM_FRAMES_PER_INTERVAL; Adapter->PollProvider = PollProviderNdis; TRY_READ_INT_CONFIGURATION(ConfigHandle, RegPollProvider, &Adapter->PollProvider); @@ -1051,6 +1061,8 @@ MpReadConfiguration( goto Exit; } + TRY_READ_INT_CONFIGURATION(ConfigHandle, RegMacLastByte, &MacLastByte); + Adapter->MACAddress[MAC_ADDR_LEN - 1] = (UCHAR)MacLastByte; Adapter->CurrentLookAhead = 0; Adapter->CurrentPacketFilter = 0; diff --git a/tools/check-drivers.ps1 b/tools/check-drivers.ps1 index fdf8df910..7c0a6c648 100644 --- a/tools/check-drivers.ps1 +++ b/tools/check-drivers.ps1 @@ -15,7 +15,10 @@ This checks for the presence of any XDP drivers currently loaded. [string]$Platform = "x64", [Parameter(Mandatory = $false)] - [switch]$IgnoreEbpf = $false + [switch]$IgnoreEbpf = $false, + + [Parameter(Mandatory = $false)] + [switch]$Force = $false ) Set-StrictMode -Version 'Latest' @@ -40,7 +43,7 @@ function Check-Driver($Driver) { function Check-And-Remove-Driver($Driver, $Component) { if (Check-Driver $Driver) { Write-Host "Detected $Driver is loaded. Uninstalling $Component..." - & $RootDir\tools\setup.ps1 -Uninstall $Component -Config $Config -Platform $Platform + & $RootDir\tools\setup.ps1 -Uninstall $Component -Config $Config -Platform $Platform -Force:$Force # Update cached driverquery output. $AllDrivers = driverquery /v /fo list @@ -56,6 +59,14 @@ function Check-And-Remove-Driver($Driver, $Component) { Check-And-Remove-Driver "xskfwdkm.sys" "xskfwdkm" Check-And-Remove-Driver "fnmp.sys" "fnmp" Check-And-Remove-Driver "fnlwf.sys" "fnlwf" +foreach ($Xdpmp in (Get-NetAdapter -IfDesc "XDPMP*" -IncludeHidden)) { + $Xdpmp.PnPDeviceID -match "^SWD\\xdpmp([0-9]+)\\" | Out-Null + $DeviceIndex = $Matches[1] + if ($DeviceIndex -ne 0) { + Write-Host "Detected XDPMP adapter with DeviceIndex $DeviceIndex. Uninstalling xdpmp..." + .\tools\setup.ps1 -Uninstall "xdpmp" -Config $Config -Platform $Platform -DeviceIndex $DeviceIndex + } +} Check-And-Remove-Driver "xdpmp.sys" "xdpmp" Check-And-Remove-Driver "xdp.sys" "xdp" Check-And-Remove-Driver "fndis.sys" "fndis" diff --git a/tools/common.ps1 b/tools/common.ps1 index 3d38e3603..fcc79d222 100644 --- a/tools/common.ps1 +++ b/tools/common.ps1 @@ -249,6 +249,43 @@ function Add-Path { Refresh-Path } +function Get-IfAliasSuffixForDeviceIndex { + param ( + [Parameter()] + [int]$DeviceIndex + ) + + if ($DeviceIndex -eq 0) { + return "" + } else { + return "$DeviceIndex" + } +} + +function Get-XdpmpIpOffsetForDeviceIndex { + param ( + [Parameter()] + [int]$DeviceIndex + ) + + return 100 + $DeviceIndex +} + +function Stop-ProcessIgnoreErrors { + param ( + [Parameter()] + [System.Diagnostics.Process]$Process + ) + + try { + if (!$Process.HasExited) { + Stop-Process -InputObject $Process -Force -ErrorAction SilentlyContinue + } + } catch { + # Ignore errors. + } +} + function Collect-LiveKD { param ( [Parameter()] diff --git a/tools/setup.ps1 b/tools/setup.ps1 index 385ddb0c1..3cd82fdaa 100644 --- a/tools/setup.ps1 +++ b/tools/setup.ps1 @@ -17,6 +17,9 @@ This script installs or uninstalls various XDP components. .PARAMETER EnableEbpf Enable eBPF in the XDP driver. + +.PARAMETER Force + Force uninstall (best effort) even if parts of the uninstall process fail. #> param ( @@ -40,6 +43,9 @@ param ( [ValidateSet("NDIS", "FNDIS")] [string]$XdpmpPollProvider = "NDIS", + [Parameter(Mandatory = $false)] + [int]$DeviceIndex = 0, + [Parameter(Mandatory = $false)] [ValidateSet("INF", "NuGet")] [string]$XdpInstaller = "NuGet", @@ -48,7 +54,10 @@ param ( [switch]$EnableEbpf = $false, [Parameter(Mandatory = $false)] - [switch]$PaLayer = $false + [switch]$PaLayer = $false, + + [Parameter(Mandatory = $false)] + [switch]$Force = $false ) Set-StrictMode -Version 'Latest' @@ -80,8 +89,10 @@ $XdpMpSys = "$ArtifactsDir\test\xdpmp\xdpmp.sys" $XdpMpInf = "$ArtifactsDir\test\xdpmp\xdpmp.inf" $XdpMpCert = "$ArtifactsDir\test\xdpmp.cer" $XdpMpComponentId = "ms_xdpmp" -$XdpMpDeviceId = "xdpmp0" $XdpMpServiceName = "XDPMP" +$XdpMpDeviceId = "xdpmp$($DeviceIndex)" +$XdpMpPnpDeviceId = "SWD\$XdpMpDeviceId\$XdpMpDeviceId" +$XdpMpIfAlias = "XDPMP" + (Get-IfAliasSuffixForDeviceIndex $DeviceIndex) $XskFwdKmSys = "$ArtifactsDir\test\xskfwdkm.sys" # Ensure the output path exists. @@ -214,14 +225,23 @@ function Install-SignedDriverCertificate($SignedFileName) { Install-DriverCertificate $CertFileName } +function Wait-For-AdapterByPnpId($PnpId, $WaitForUp=$true) { + Write-Verbose "Waiting for `"$PnpId`" adapter to start" + $Filter = { $_.PnpDeviceID -eq "$PnpId" -and (!$WaitForUp -or $_.Status -eq "Up") } + return Wait-For-Adapters $Filter 1 $WaitForUp +} + # Helper to wait for an adapter to start. -function Wait-For-Adapters($IfDesc, $Count=1, $WaitForUp=$true) { - Write-Verbose "Waiting for $Count `"$IfDesc`" adapter(s) to start" +function Wait-For-Adapters($Filter, $Count=1, $WaitForUp=$true) { + Write-Verbose "Waiting for $Count adapter(s) to start" $StartSuccess = $false + $Results = $null for ($i = 0; $i -lt 100; $i++) { $Result = 0 - $Filter = { $_.InterfaceDescription -like "$IfDesc*" -and (!$WaitForUp -or $_.Status -eq "Up") } - try { $Result = ((Get-NetAdapter | where $Filter) | Measure-Object).Count } catch {} + try { + $Results = Get-NetAdapter | where $Filter + $Result = ($Results | Measure-Object).Count + } catch {} if ($Result -eq $Count) { $StartSuccess = $true break; @@ -230,9 +250,10 @@ function Wait-For-Adapters($IfDesc, $Count=1, $WaitForUp=$true) { } if ($StartSuccess -eq $false) { Get-NetAdapter | Format-Table | Out-String | Write-Verbose - Write-Error "Failed to start $Count `"$IfDesc`" adapters(s) [$Result/$Count]" + Write-Error "Failed to start $Count adapters(s) [$Result/$Count]" } else { - Write-Verbose "Started $Count `"$IfDesc`" adapter(s)" + Write-Verbose "Started $Count adapter(s)" + return $Results } } @@ -418,12 +439,14 @@ function Install-XdpMp { Write-Error "$XdpMpSys does not exist!" } - Install-DriverCertificate $XdpMpCert + if ($DeviceIndex -eq 0) { + Install-DriverCertificate $XdpMpCert - Write-Verbose "pnputil.exe /install /add-driver $XdpMpInf" - pnputil.exe /install /add-driver $XdpMpInf | Write-Verbose - if ($LastExitCode) { - Write-Error "pnputil.exe exit code: $LastExitCode" + Write-Verbose "pnputil.exe /install /add-driver $XdpMpInf" + pnputil.exe /install /add-driver $XdpMpInf | Write-Verbose + if ($LastExitCode) { + Write-Error "pnputil.exe exit code: $LastExitCode" + } } Write-Verbose "dswdevice.exe -i $XdpMpDeviceId $XdpMpComponentId" @@ -434,19 +457,22 @@ function Install-XdpMp { # Do not wait for the adapter to fully come up: the default is NDIS polling, # which is not available prior to WS2022. - Wait-For-Adapters -IfDesc $XdpMpServiceName -WaitForUp $false + $Adapter = Wait-For-AdapterByPnpId -PnpId $XdpMpPnpDeviceId -WaitForUp $false Write-Verbose "Renaming adapter" - Rename-NetAdapter-With-Retry $XdpMpServiceName $XdpMpServiceName + Rename-NetAdapter-With-Retry $Adapter.InterfaceDescription $XdpMpIfAlias - Write-Verbose "Get-NetAdapter $XdpMpServiceName" - Get-NetAdapter $XdpMpServiceName | Format-Table | Out-String | Write-Verbose - $AdapterIndex = (Get-NetAdapter $XdpMpServiceName).ifIndex + Write-Verbose "Get-NetAdapter $XdpMpIfAlias" + Get-NetAdapter $XdpMpIfAlias | Format-Table | Out-String | Write-Verbose + $AdapterIndex = (Get-NetAdapter $XdpMpIfAlias).ifIndex Write-Verbose "Setting up the adapter" - Write-Verbose "Set-NetAdapterAdvancedProperty -Name $XdpMpServiceName -RegistryKeyword PollProvider -DisplayValue $XdpmpPollProvider" - Set-NetAdapterAdvancedProperty -Name $XdpMpServiceName -RegistryKeyword PollProvider -DisplayValue $XdpmpPollProvider + Write-Verbose "Set-NetAdapterAdvancedProperty -Name $XdpMpIfAlias -RegistryKeyword PollProvider -DisplayValue $XdpmpPollProvider" + Set-NetAdapterAdvancedProperty -Name $XdpMpIfAlias -RegistryKeyword PollProvider -DisplayValue $XdpmpPollProvider + + Write-Verbose "Set-NetAdapterAdvancedProperty -Name $XdpMpIfAlias -RegistryKeyword MACLastByte -DisplayValue $DeviceIndex" + Set-NetAdapterAdvancedProperty -Name $XdpMpIfAlias -RegistryKeyword MACLastByte -DisplayValue $DeviceIndex if ($XdpmpPollProvider -eq "NDIS") { #Write-Verbose "Set-NetAdapterDataPathConfiguration -Name $XdpMpServiceName -Profile Passive" @@ -454,27 +480,28 @@ function Install-XdpMp { Write-Verbose "Skipping NDIS polling configuration" } - Wait-For-Adapters -IfDesc $XdpMpServiceName + Wait-For-AdapterByPnpId -PnpId $XdpMpPnpDeviceId -WaitForUp $false | Out-Null - netsh.exe int ipv4 set int $AdapterIndex dadtransmits=0 | Write-Verbose - netsh.exe int ipv4 add address $AdapterIndex address=192.168.100.1/24 | Write-Verbose - netsh.exe int ipv4 add neighbor $AdapterIndex address=192.168.100.2 neighbor=22-22-22-22-00-02 | Write-Verbose + $IpOffset = Get-XdpmpIpOffsetForDeviceIndex $DeviceIndex + netsh.exe int ipv4 set int $AdapterIndex dadtransmits=0 | Write-Verbose + netsh.exe int ipv4 add address $AdapterIndex address=192.168.$IpOffset.1/24 | Write-Verbose + netsh.exe int ipv4 add neighbor $AdapterIndex address=192.168.$IpOffset.2 neighbor=22-22-22-22-00-02 | Write-Verbose netsh.exe int ipv6 set int $AdapterIndex dadtransmits=0 | Write-Verbose - netsh.exe int ipv6 add address $AdapterIndex address=fc00::100:1/112 | Write-Verbose - netsh.exe int ipv6 add neighbor $AdapterIndex address=fc00::100:2 neighbor=22-22-22-22-00-02 | Write-Verbose + netsh.exe int ipv6 add address $AdapterIndex address=fc00::$($IpOffset):1/112 | Write-Verbose + netsh.exe int ipv6 add neighbor $AdapterIndex address=fc00::$($IpOffset):2 neighbor=22-22-22-22-00-02 | Write-Verbose Write-Verbose "Adding firewall rules" - netsh.exe advfirewall firewall add rule name="Allow$($XdpMpServiceName)v4" dir=in action=allow protocol=any remoteip=192.168.100.0/24 | Write-Verbose - netsh.exe advfirewall firewall add rule name="Allow$($XdpMpServiceName)v6" dir=in action=allow protocol=any remoteip=fc00::100:0/112 | Write-Verbose + netsh.exe advfirewall firewall add rule name="Allow$($XdpMpIfAlias)v4" dir=in action=allow protocol=any remoteip=192.168.$IpOffset.0/24 | Write-Verbose + netsh.exe advfirewall firewall add rule name="Allow$($XdpMpIfAlias)v6" dir=in action=allow protocol=any remoteip=fc00::$($IpOffset):0/112 | Write-Verbose Write-Verbose "xdpmp.sys install complete!" } # Uninstalls the xdpmp driver. function Uninstall-XdpMp { - netsh.exe advfirewall firewall del rule name="Allow$($XdpMpServiceName)v4" | Out-Null - netsh.exe advfirewall firewall del rule name="Allow$($XdpMpServiceName)v6" | Out-Null + netsh.exe advfirewall firewall del rule name="Allow$($XdpMpIfAlias)v4" | Out-Null + netsh.exe advfirewall firewall del rule name="Allow$($XdpMpIfAlias)v6" | Out-Null Write-Verbose "$DswDevice -u $XdpMpDeviceId" cmd.exe /c "$DswDevice -u $XdpMpDeviceId 2>&1" | Write-Verbose @@ -482,15 +509,17 @@ function Uninstall-XdpMp { Write-Host "Deleting $XdpMpDeviceId device failed: $LastExitCode" } - Write-Verbose "$DevCon remove @SWD\$XdpMpDeviceId\$XdpMpDeviceId" - cmd.exe /c "$DevCon remove @SWD\$XdpMpDeviceId\$XdpMpDeviceId 2>&1" | Write-Verbose + Write-Verbose "$DevCon remove @$XdpMpPnpDeviceId" + cmd.exe /c "$DevCon remove @$XdpMpPnpDeviceId 2>&1" | Write-Verbose if (!$?) { Write-Host "Removing $XdpMpDeviceId device failed: $LastExitCode" } - Cleanup-Service $XdpMpServiceName + if ($DeviceIndex -eq 0) { + Cleanup-Service $XdpMpServiceName - Uninstall-Driver "xdpmp.inf" + Uninstall-Driver "xdpmp.inf" + } Write-Verbose "xdpmp.sys uninstall complete!" } @@ -628,11 +657,23 @@ function Uninstall-Ebpf { } if ($Process.ExitCode -ne 0) { + if (!$Force) { + Write-Error "Uninstalling eBPF from $EbpfMsiFullPath failed: $($Process.ExitCode)" + } + Write-Warning "Uninstalling eBPF from $EbpfMsiFullPath failed: $($Process.ExitCode)" if ($Process.ExitCode -eq 0x645) { Write-Warning "eBPF is present but the MSI is not installed. Trying to uninstall services and binaries..." + Write-Verbose "Stop-Service ebpfsvc" + try { Stop-Service ebpfsvc -NoWait } catch { } + Cleanup-Service ebpfsvc + + Write-Verbose "Stop-Service netebpfext" + try { Stop-Service netebpfext -NoWait } catch { } + Cleanup-Service netebpfext + Write-Verbose "Stop-Service netebpfext" try { Stop-Service netebpfext -NoWait } catch { } Cleanup-Service netebpfext diff --git a/tools/spinxsk.ps1 b/tools/spinxsk.ps1 index b4a6214b1..6e46849c6 100644 --- a/tools/spinxsk.ps1 +++ b/tools/spinxsk.ps1 @@ -18,6 +18,9 @@ more coverage for setup and cleanup. .PARAMETER Stats Periodic socket statistics output. +.PARAMETER InterfaceCount + Number of interfaces to create and spin. + .PARAMETER QueueCount Number of queues to spin. @@ -65,6 +68,9 @@ param ( [Parameter(Mandatory = $false)] [switch]$Stats = $false, + [Parameter(Mandatory = $false)] + [Int32]$InterfaceCount = 1, + [Parameter(Mandatory = $false)] [Int32]$QueueCount = 2, @@ -120,6 +126,8 @@ $LogsDir = "$RootDir\artifacts\logs" $SpinXsk = "$ArtifactsDir\test\spinxsk.exe" $LiveKD = Get-CoreNetCiArtifactPath -Name "livekd64.exe" $KD = Get-CoreNetCiArtifactPath -Name "kd.exe" +$BpfSourceDir = "$ArtifactsDir\test\bpf" +$SpinxskWorkDir = "$RootDir\artifacts\spinxsk" # Verify all the files are present. if (!(Test-Path $SpinXsk)) { @@ -130,10 +138,12 @@ if (!(Test-Path $SpinXsk)) { New-Item -ItemType Directory -Force -Path $LogsDir | Out-Null $StartTime = Get-Date -$WsaRioProcess = $null $WsaRio = Get-CoreNetCiArtifactPath -Name "wsario.exe" while (($Minutes -eq 0) -or (((Get-Date)-$StartTime).TotalMinutes -lt $Minutes)) { + $WsaRioProcesses = @() + $SpinxskProcesses = @() + $SpinxskInstanceDirs = @() $ThisIterationMinutes = 10 if ($Minutes -ne 0) { @@ -164,6 +174,7 @@ while (($Minutes -eq 0) -or (((Get-Date)-$StartTime).TotalMinutes -lt $Minutes)) } if (!$EbpfPreinstalled) { + # Always install eBPF because spinxsk links to the eBPF DLL. Write-Verbose "installing ebpf..." & "$RootDir\tools\setup.ps1" -Install ebpf -Config $Config -Platform $Platform Write-Verbose "installed ebpf." @@ -173,91 +184,136 @@ while (($Minutes -eq 0) -or (((Get-Date)-$StartTime).TotalMinutes -lt $Minutes)) & "$RootDir\tools\setup.ps1" -Install xdp -Config $Config -Platform $Platform -EnableEbpf:$EnableEbpf -XdpInstaller $XdpInstaller Write-Verbose "installed xdp." - if ($Driver -eq "XDPMP") { - Write-Verbose "installing xdpmp..." - & "$RootDir\tools\setup.ps1" -Install xdpmp -XdpmpPollProvider $XdpmpPollProvider -Config $Config -Platform $Platform - Write-Verbose "installed xdpmp." + Write-Verbose "reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\xdp\Parameters /v XdpFaultInject /d 1 /t REG_DWORD /f" + reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\xdp\Parameters /v XdpFaultInject /d 1 /t REG_DWORD /f | Write-Verbose - $AdapterName = "XDPMP" - } else { - Write-Verbose "installing fnmp..." - & "$RootDir\tools\setup.ps1" -Install fnmp -Config $Config -Platform $Platform - Write-Verbose "installed fnmp." + $InterfaceIndices = @() - $AdapterName = "XDPFNMP" - } + for ($i = 0; $i -lt $InterfaceCount; $i++) { + if ($Driver -eq "XDPMP") { + Write-Verbose "installing xdpmp instance $i..." + & "$RootDir\tools\setup.ps1" -Install xdpmp -XdpmpPollProvider $XdpmpPollProvider -Config $Config -Platform $Platform -DeviceIndex $i + Write-Verbose "installed xdpmp instance $i." - Write-Verbose "Set-NetAdapterRss $AdapterName -NumberOfReceiveQueues $QueueCount" - Set-NetAdapterRss $AdapterName -NumberOfReceiveQueues $QueueCount + $AdapterName = "XDPMP" + } else { + Write-Verbose "installing fnmp instance $i..." + & "$RootDir\tools\setup.ps1" -Install fnmp -Config $Config -Platform $Platform -DeviceIndex $i + Write-Verbose "installed fnmp instance $i." - Write-Verbose "reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\xdp\Parameters /v XdpFaultInject /d 1 /t REG_DWORD /f" - reg.exe add HKLM\SYSTEM\CurrentControlSet\Services\xdp\Parameters /v XdpFaultInject /d 1 /t REG_DWORD /f | Write-Verbose + $AdapterName = "XDPFNMP" + } - $Args = ` - "-IfIndex", (Get-NetAdapter $AdapterName).ifIndex, ` - "-QueueCount", $QueueCount, ` - "-Minutes", $ThisIterationMinutes, ` - "-GlobalConcurrentWorkers", $GlobalConcurrentWorkerCount - if ($Stats) { - $Args += "-Stats" - } - if ($FuzzerCount -ne 0) { - $Args += "-FuzzerCount", $FuzzerCount - } - if ($CleanDatapath) { - $Args += "-CleanDatapath" - } - if ($SuccessThresholdPercent -ge 0) { - $Args += "-SuccessThresholdPercent", $SuccessThresholdPercent - } - if ($BreakOnWatchdog) { - $Args += "-WatchdogCmd", "break" - } else { - $Args += "-WatchdogCmd", "$LiveKD -o $LogsDir\spinxsk_watchdog.dmp -k $KD -ml -accepteula" - } - if ($EnableEbpf) { - $Args += "-EnableEbpf" - } - if ($Driver -eq "FNMP") { - $Args += "-UseFnmp" + $AdapterName += (Get-IfAliasSuffixForDeviceIndex $i) + $InterfaceIndices += (Get-NetAdapter -Name $AdapterName).ifIndex + + Write-Verbose "Set-NetAdapterRss $AdapterName -NumberOfReceiveQueues $QueueCount" + Set-NetAdapterRss $AdapterName -NumberOfReceiveQueues $QueueCount } - if ($TxInspect) { - - # Assuming 192.168.100.2:1234 is the XDPMP IP address based on snippet in xskperf.ps1: - # if ($Adapter.InterfaceDescription -like "XDPMP*") { - # $RemoteAddress = "192.168.100.2:1234" - # } - - $XskGroup = 0 - $IoSize = 64 - $UdpSize = $IoSize - 8 - 20 - 14 - $TxInspectContentionCount = 2 - $ArgList = - "Winsock Send -Target 192.168.100.2:1234 -Group $XskGroup " + - "-IoSize $UdpSize -IoCount -1 -ThreadCount $TxInspectContentionCount" - Write-Verbose "$WsaRio $ArgList" - $WsaRioProcess = Start-Process $WsaRio -PassThru -ArgumentList $ArgList + for ($i = 0; $i -lt $InterfaceCount; $i++) { + $SpinxskArgs = ` + "-IfIndex", $InterfaceIndices[$i], ` + "-QueueCount", $QueueCount, ` + "-Minutes", $ThisIterationMinutes, ` + "-GlobalConcurrentWorkers", $GlobalConcurrentWorkerCount + if ($Stats) { + $SpinxskArgs += "-Stats" + } + if ($FuzzerCount -ne 0) { + $SpinxskArgs += "-FuzzerCount", $FuzzerCount + } + if ($CleanDatapath) { + $SpinxskArgs += "-CleanDatapath" + } + if ($SuccessThresholdPercent -ge 0) { + $SpinxskArgs += "-SuccessThresholdPercent", $SuccessThresholdPercent + } + if ($BreakOnWatchdog) { + $SpinxskArgs += "-WatchdogCmd", "break" + } else { + $SpinxskArgs += "-WatchdogCmd", "`"$LiveKD -o $LogsDir\spinxsk_watchdog.dmp -k $KD -ml -accepteula`"" + } + if ($EnableEbpf) { + $SpinxskArgs += "-EnableEbpf" + + $InstanceDir = "$SpinxskWorkDir\instance_$i" + New-Item -ItemType Directory -Force -Path $InstanceDir | Out-Null + Copy-Item -Path "$BpfSourceDir\*" -Destination $InstanceDir -Recurse -Force + $SpinxskArgs += "-BpfDir", $InstanceDir + $SpinxskInstanceDirs += $InstanceDir + } + if ($Driver -eq "FNMP") { + $SpinxskArgs += "-UseFnmp" + } + + if ($TxInspect) { + + # Assuming 192.168.100.2:1234 is the XDPMP IP address based on snippet in xskperf.ps1: + # if ($Adapter.InterfaceDescription -like "XDPMP*") { + # $RemoteAddress = "192.168.100.2:1234" + # } + + $XskGroup = 0 + $IoSize = 64 + $UdpSize = $IoSize - 8 - 20 - 14 + $TxInspectContentionCount = 2 + $ArgList = + "Winsock Send -Target 192.168.$(Get-XdpmpIpOffsetForDeviceIndex $i).2:1234 -Group $XskGroup " + + "-IoSize $UdpSize -IoCount -1 -ThreadCount $TxInspectContentionCount" + Write-Verbose "$WsaRio $ArgList" + $WsaRioProcesses += Start-Process $WsaRio -PassThru -ArgumentList $ArgList + } + + $StartInfo = New-Object System.Diagnostics.ProcessStartInfo + $StartInfo.FileName = $SpinXsk + $StartInfo.RedirectStandardError = $true + $StartInfo.RedirectStandardOutput = $true + $StartInfo.UseShellExecute = $false + $StartInfo.Arguments = $SpinxskArgs + $SpinxskProcess = New-Object System.Diagnostics.Process + $SpinxskProcess.StartInfo = $StartInfo + + Write-Verbose "$SpinXsk $SpinxskArgs" + $SpinxskProcess.Start() | Out-Null + $SpinxskProcesses += $SpinxskProcess } - Write-Verbose "$SpinXsk $Args" - & $SpinXsk $Args - if ($LastExitCode -ne 0) { - throw "SpinXsk failed with $LastExitCode" + foreach ($SpinxskProcess in $SpinxskProcesses) { + Write-Verbose "Waiting for SpinXsk process ID $($SpinxskProcess.Id)..." + $SpinxskProcess.WaitForExit() + $StdOutput = $SpinxskProcess.StandardOutput.ReadToEnd() + $StdError = $SpinxskProcess.StandardError.ReadToEnd() + if (![string]::IsNullOrWhiteSpace($StdOutput)) { + Write-Verbose "stdout:`n$StdOutput" + } + if (![string]::IsNullOrWhiteSpace($StdError)) { + Write-Warning "stderr:`n$StdError" + } + if ($SpinxskProcess.ExitCode -ne 0) { + throw "SpinXsk failed with $($SpinxskProcess.ExitCode)" + } } } catch { Write-Error "Error: $_" exit 1 } finally { - if ($Driver -eq "XDPMP") { - & "$RootDir\tools\setup.ps1" -Uninstall xdpmp -Config $Config -Platform $Platform -ErrorAction 'Continue' - } else { - & "$RootDir\tools\setup.ps1" -Uninstall fnmp -Config $Config -Platform $Platform -ErrorAction 'Continue' + foreach ($SpinxskProcess in $SpinxskProcesses) { + Write-Verbose "Stopping SpinXsk process ID $($SpinxskProcess.Id)..." + Stop-ProcessIgnoreErrors $SpinxskProcess + } + for ($i = $InterfaceCount - 1; $i -ge 0; $i--) { + & "$RootDir\tools\setup.ps1" -Uninstall $Driver -Config $Config -Platform $Platform -DeviceIndex $i -ErrorAction 'Continue' } & "$RootDir\tools\setup.ps1" -Uninstall xdp -Config $Config -Platform $Platform -XdpInstaller $XdpInstaller -ErrorAction 'Continue' if (!$EbpfPreinstalled) { & "$RootDir\tools\setup.ps1" -Uninstall ebpf -Config $Config -Platform $Platform -ErrorAction 'Continue' } + foreach ($InstanceDir in $SpinxskInstanceDirs) { + if (Test-Path $InstanceDir) { + Remove-Item -Path $InstanceDir -Recurse -Force -ErrorAction 'Continue' + } + } if ($XdpmpPollProvider -eq "FNDIS") { & "$RootDir\tools\setup.ps1" -Uninstall fndis -Config $Config -Platform $Platform -ErrorAction 'Continue' } @@ -269,8 +325,9 @@ while (($Minutes -eq 0) -or (((Get-Date)-$StartTime).TotalMinutes -lt $Minutes)) & "$RootDir\tools\log.ps1" -Stop -Name spinxskebpf_$Driver -Config $Config -Platform $Platform -ErrorAction 'Continue' & "$RootDir\tools\log.ps1" -Stop -Name spinxsk_$Driver -Config $Config -Platform $Platform -ErrorAction 'Continue' } - if ($WsaRioProcess) { - Stop-Process -InputObject $WsaRioProcess + foreach ($WsaRioProcess in $WsaRioProcesses) { + Write-Verbose "Stopping WsaRio..." + Stop-ProcessIgnoreErrors $WsaRioProcess } } }