From da4ac3e10ef2d9cef00da6136a9830c9658c56ce Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Wed, 7 Jan 2026 15:26:17 +0100 Subject: [PATCH 01/16] Add circular iterators, use them in the border router. --- pkg/slices/BUILD.bazel | 10 +- pkg/slices/iterator.go | 75 +++++++++ pkg/slices/iterator_test.go | 187 +++++++++++++++++++++ router/metrics.go | 5 +- router/underlayproviders/udpip/BUILD.bazel | 1 + router/underlayproviders/udpip/udpip.go | 136 ++++++++++----- 6 files changed, 365 insertions(+), 49 deletions(-) create mode 100644 pkg/slices/iterator.go create mode 100644 pkg/slices/iterator_test.go diff --git a/pkg/slices/BUILD.bazel b/pkg/slices/BUILD.bazel index f5beed85b0..99afad5905 100644 --- a/pkg/slices/BUILD.bazel +++ b/pkg/slices/BUILD.bazel @@ -3,14 +3,20 @@ load("//tools:go.bzl", "go_test") go_library( name = "go_default_library", - srcs = ["transform.go"], + srcs = [ + "iterator.go", + "transform.go", + ], importpath = "github.com/scionproto/scion/pkg/slices", visibility = ["//visibility:public"], ) go_test( name = "go_default_test", - srcs = ["transform_test.go"], + srcs = [ + "iterator_test.go", + "transform_test.go", + ], deps = [ ":go_default_library", "@com_github_stretchr_testify//assert:go_default_library", diff --git a/pkg/slices/iterator.go b/pkg/slices/iterator.go new file mode 100644 index 0000000000..7a2f48226b --- /dev/null +++ b/pkg/slices/iterator.go @@ -0,0 +1,75 @@ +// Copyright 2025 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package slices + +import ( + "iter" +) + +type Iter[T any] iter.Seq2[int, T] + +func CircularTransformingIterator[IN, OUT any]( + s []IN, + first int, + count int, + transform func(index int) OUT, +) Iter[OUT] { + // XXX(juagargi): beware of the uint casts. They are necessary for the mod operation (down in + // the for loop) to be efficient. See also the benchmarks' results. + L := uint(len(s)) + if first < 0 { + m := (-first)/int(L) + 1 + first += m * int(L) + } else { + first = first % int(L) + } + return func(yield func(int, OUT) bool) { + first := uint(first) + count := uint(count) + for i := uint(0); i < count; i++ { + idx := (i + first) % L + if !yield(int(idx), transform(int(idx))) { + return + } + } + } +} + +// CircularIterator creates a push iterator for the slice that starts at `first` and ends after +// `count` elements. This iterator can be directly used in for-range loops. +func CircularIterator[T any](s []T, first int, count int) Iter[T] { + return CircularTransformingIterator(s, first, count, func(index int) T { + return s[index] + }) +} + +// CDIterator returns a Circular Dereferencing Iterator, similarly to CircularIterator, +// but each element is the pointer to the original element in the slice. +func CDIterator[T any](s []T, first int, count int) Iter[*T] { + return CircularTransformingIterator(s, first, count, func(index int) *T { + return &s[index] + }) +} + +// ToValueIterator adapts a index-value push iterator to a value push iterator. +func ToValueIterator[T any](it Iter[T]) iter.Seq[T] { + return func(yield func(T) bool) { + for _, v := range it { + if !yield(v) { + return + } + } + } +} diff --git a/pkg/slices/iterator_test.go b/pkg/slices/iterator_test.go new file mode 100644 index 0000000000..fb75f54ffc --- /dev/null +++ b/pkg/slices/iterator_test.go @@ -0,0 +1,187 @@ +// Copyright 2025 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package slices_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/scionproto/scion/pkg/slices" +) + +func TestModulusIterator(t *testing.T) { + // Initialize base case. + s := rangeSlice(0, 10) // 0, 1, ... , 8, 9. + for i := range s { + s[i] *= 10 // 0, 10, 20, ... , 80, 90. + } + + cases := map[string]struct { + first int + count int + expected []int + }{ + "empty": { + first: 0, + count: 0, + expected: []int{}, + }, + "full": { + first: 0, + count: len(s), + expected: rangeSlice(0, 10), + }, + "linear": { + first: 1, + count: 2, + expected: []int{1, 2}, + }, + "gap": { + first: 9, + count: 1, + expected: []int{9}, + }, + "discontinuity": { + first: 9, + count: 2, + expected: []int{9, 0}, + }, + "negative_index": { + first: -1, + count: 2, + expected: []int{9, 0}, + }, + "too_large_index": { + first: 2*len(s) + 9, + count: 2, + expected: []int{9, 0}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + got := []int{} + iterator := slices.CircularIterator(s, tc.first, tc.count) + for i, v := range iterator { + t.Logf("s[%d] = %v\n", i, v) + got = append(got, i) + } + assert.Equal(t, tc.expected, got) + }) + } +} + +// BenchmarkIterators checks that the performance of the iterators is similar to that of a regular +// for-loop. Deviations of 5% (on both directions, sometimes iterators are faster) are expected. +func BenchmarkIterators(b *testing.B) { + const benchmarkSliceSize = 1024 * 1024 + + b.Run("regular", func(b *testing.B) { + b.Run("index-value", func(b *testing.B) { + collection := make([]int, benchmarkSliceSize) + sum := 0 + b.ResetTimer() + for range b.N { + for i, v := range collection { + sum += v + _ = i + } + } + }) + + b.Run("index", func(b *testing.B) { + collection := make([]int, benchmarkSliceSize) + sum := 0 + b.ResetTimer() + for range b.N { + for i := range collection { + sum += collection[i] + } + } + }) + + b.Run("value", func(b *testing.B) { + collection := make([]int, benchmarkSliceSize) + sum := 0 + b.ResetTimer() + for range b.N { + for _, v := range collection { + sum += v + } + } + }) + }) + + b.Run("iterators", func(b *testing.B) { + b.Run("circular-value", func(b *testing.B) { + collection := make([]int, benchmarkSliceSize) + sum := 0 + b.ResetTimer() + for range b.N { + iterator := slices.CircularIterator(collection, 1, benchmarkSliceSize) + for _, v := range iterator { + sum += v + } + } + }) + + b.Run("circular-index", func(b *testing.B) { + collection := make([]int, benchmarkSliceSize) + sum := 0 + b.ResetTimer() + for range b.N { + iterator := slices.CircularIterator(collection, 1, benchmarkSliceSize) + for i := range iterator { + sum += collection[i] + } + } + }) + + b.Run("cditerator", func(b *testing.B) { + collection := make([]int, benchmarkSliceSize) + sum := 0 + b.ResetTimer() + for range b.N { + iterator := slices.CDIterator(collection, 1, benchmarkSliceSize) + for _, v := range iterator { + sum += *v + } + } + }) + + b.Run("tovalue", func(b *testing.B) { + collection := make([]int, benchmarkSliceSize) + sum := 0 + b.ResetTimer() + for range b.N { + iterator := slices.ToValueIterator( + slices.CircularIterator(collection, 1, benchmarkSliceSize)) + for v := range iterator { + sum += v + } + } + }) + }) +} + +func rangeSlice(begin, end int) []int { + s := make([]int, end-begin) + for i := begin; i < end; i++ { + s[i] = i + } + return s +} diff --git a/router/metrics.go b/router/metrics.go index e896db10c4..426954ba8c 100644 --- a/router/metrics.go +++ b/router/metrics.go @@ -16,6 +16,7 @@ package router import ( + "iter" "math/bits" "strconv" "strings" @@ -399,13 +400,13 @@ func serviceLabels(localIA addr.IA, svc addr.SVC) prometheus.Labels { // UpdateOutputMetrics updates the given InterfaceMetrics in bulk according // to the given set of just sent packets. This is much faster than looking up // the right set of metrics by size class and traffic type for each packet. -func UpdateOutputMetrics(metrics *InterfaceMetrics, packets []*Packet) { +func UpdateOutputMetrics(metrics *InterfaceMetrics, packets iter.Seq[*Packet]) { // We need to collect stats by traffic type and size class. // Try to reduce the metrics lookup penalty by using some // simpler staging data structure. writtenPkts := [ttMax][maxSizeClass]int{} writtenBytes := [ttMax][maxSizeClass]int{} - for _, p := range packets { + for p := range packets { s := len(p.RawPacket) sc := ClassOfSize(s) tt := p.trafficType diff --git a/router/underlayproviders/udpip/BUILD.bazel b/router/underlayproviders/udpip/BUILD.bazel index db7afc0108..3437619762 100644 --- a/router/underlayproviders/udpip/BUILD.bazel +++ b/router/underlayproviders/udpip/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//pkg/private/serrors:go_default_library", "//pkg/slayers:go_default_library", "//pkg/stun:go_default_library", + "//pkg/slices:go_default_library", "//private/underlay/conn:go_default_library", "//router:go_default_library", "//router/bfd:go_default_library", diff --git a/router/underlayproviders/udpip/udpip.go b/router/underlayproviders/udpip/udpip.go index f5b12da0cd..960e008663 100644 --- a/router/underlayproviders/udpip/udpip.go +++ b/router/underlayproviders/udpip/udpip.go @@ -31,6 +31,7 @@ import ( "github.com/scionproto/scion/pkg/log" "github.com/scionproto/scion/pkg/private/serrors" "github.com/scionproto/scion/pkg/slayers" + sslices "github.com/scionproto/scion/pkg/slices" "github.com/scionproto/scion/pkg/stun" "github.com/scionproto/scion/private/underlay/conn" "github.com/scionproto/scion/router" @@ -207,7 +208,7 @@ type udpConnection struct { name string // for logs. It's more informative than ifID. link udpLink // Link with exclusive use of the connection. links map[netip.AddrPort]udpLink // Links that share this connection - queue chan *router.Packet + queue chan *router.Packet // Packets to be sent. metrics *router.InterfaceMetrics receiverDone chan struct{} senderDone chan struct{} @@ -315,35 +316,65 @@ func (u *udpConnection) receive(batchSize int, pool router.PacketPool) { } } -func readUpTo(queue <-chan *router.Packet, n int, needsBlocking bool, pkts []*router.Packet) int { - i := 0 - if needsBlocking { - p, ok := <-queue - if !ok { - return i +func readUpTo( + pktIter sslices.Iter[**router.Packet], // Where to write the packet pointer. + queue <-chan *router.Packet, // Packet pointer source. + needsBlocking bool, +) int { + // This is the reading function pointer. + var read func(**router.Packet) bool + + // This reading function implementation changes the function pointer to a + // simpler one after the first run. This allows to check for blocking behavior only once. + // After the first call to read(), read will always behave as readAsync. + read = func(ptr **router.Packet) bool { + readBlock := func(ptr **router.Packet) bool { + var ok bool + *ptr, ok = <-queue + return ok } - pkts[i] = p - i++ - } - for ; i < n; i++ { - select { - case p, ok := <-queue: - if !ok { - return i + readAsync := func(ptr **router.Packet) bool { + var ok bool + select { + case *ptr, ok = <-queue: + default: } - pkts[i] = p - default: - return i + return ok + } + + // Modify the function pointer to readAsync. + read = readAsync + + // And for the first time and only time this function runs, call block or async. + var ok bool + if needsBlocking { + ok = readBlock(ptr) + } else { + ok = readAsync(ptr) } + return ok } - return i + + pktCount := 0 + for _, ptr := range pktIter { + if !read(ptr) { + break + } + pktCount++ + } + return pktCount } func (u *udpConnection) send(batchSize int, pool router.PacketPool) { log.Debug("Send", "connection", u.name) - // We use this somewhat like a ring buffer. + // Ring buffer storing the packets. + // Using circular (modular) iterators to access this buffer. + // Depiction of the ring buffer: + // |x|x| | | | |x|x| + // With x meaning packet to be sent on that index. + // The buffer above has batchSize = 8, currentIdx = 6, toWrite = 4. pkts := make([]*router.Packet, batchSize) // We use this as a temporary buffer, but allocate it just once @@ -353,47 +384,62 @@ func (u *udpConnection) send(batchSize int, pool router.PacketPool) { msgs[i].Buffers = make([][]byte, 1) } - queue := u.queue - conn := u.conn - metrics := u.metrics - toWrite := 0 - + currentIdx := 0 // Index of the first packet pending to be sent. + toWrite := 0 // Amount of packets pending to be sent. for u.running.Load() { - // Top-up our batch. - toWrite += readUpTo(queue, batchSize-toWrite, toWrite == 0, pkts[toWrite:]) + // Top-up our batch. Write onto the ring buffer, starting from the first free "bucket" and + // no more than the count of free buckets. + newBatchPktCount := readUpTo( + sslices.CDIterator(pkts, currentIdx+toWrite, batchSize-toWrite), + u.queue, + toWrite == 0) // Turn the packets into underlay messages that WriteBatch can send. - for i, p := range pkts[:toWrite] { + // Only packets stored from currentIdx+toWrite and onwards are new, copy only the new ones. + i := 0 + for _, p := range sslices.CircularIterator(pkts, currentIdx, toWrite+newBatchPktCount) { msgs[i].Buffers[0] = p.RawPacket - msgs[i].Addr = nil - // If we're using a connected socket we must not specify the address. It might cause - // redundant route queries and the address might not even be set in the packet. - // Otherwise, we must specify the address. - if !u.connected { + if u.connected { + // If we're using a connected socket we must not specify the address. It might cause + // redundant route queries and the address might not even be set in the packet. + msgs[i].Addr = nil + } else { + // Otherwise, we must specify the address. msgs[i].Addr = (*net.UDPAddr)(p.RemoteAddr) } + i++ } - written, _ := conn.WriteBatch(msgs[:toWrite], 0) + // Attempt to write the remaining packets from previous batches and this new one. + written, _ := u.conn.WriteBatch(msgs[:toWrite+newBatchPktCount], 0) if written < 0 { // WriteBatch returns -1 on error, we just consider this as - // 0 packets written + // 0 packets written. written = 0 } - router.UpdateOutputMetrics(metrics, pkts[:written]) - for _, p := range pkts[:written] { + iterator := sslices.ToValueIterator( + sslices.CircularIterator(pkts, currentIdx, written)) + router.UpdateOutputMetrics(u.metrics, iterator) + // Return storage for all the written packets. + for p := range iterator { pool.Put(p) } + // The next packet to write is now the first one not written. + currentIdx = (currentIdx + written) % batchSize + + // Compute the number of packets to still write for next iteration. + toWrite += newBatchPktCount if written != toWrite { - // Only one is dropped at this time. We'll retry the rest. - sc := router.ClassOfSize(len(pkts[written].RawPacket)) - metrics[sc].DroppedPacketsInvalid.Inc() - pool.Put(pkts[written]) + // The batch was not completely written. We assume that the failure was caused by + // the first packet not being sent, i.e. with index = currentIdx. + taintedPktIndex := currentIdx + sc := router.ClassOfSize(len(pkts[taintedPktIndex].RawPacket)) + u.metrics[sc].DroppedPacketsInvalid.Inc() + // Return storage for this bad packet. + pool.Put(pkts[taintedPktIndex]) + // We drop the packet and try again with the rest. + currentIdx++ toWrite -= (written + 1) - // Shift the leftovers to the head of the buffers. - for i := 0; i < toWrite; i++ { - pkts[i] = pkts[i+written+1] - } } else { toWrite = 0 } From d09ea7a3553f9cf064fc5b461cf34a8d7b6c03bd Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Thu, 8 Jan 2026 13:22:37 +0100 Subject: [PATCH 02/16] Add priority-aware read functions. --- router/priority/BUILD.bazel | 19 +++ router/priority/export_test.go | 26 +++++ router/priority/priority.go | 111 ++++++++++++++++++ router/priority/priority_test.go | 191 +++++++++++++++++++++++++++++++ 4 files changed, 347 insertions(+) create mode 100644 router/priority/BUILD.bazel create mode 100644 router/priority/export_test.go create mode 100644 router/priority/priority.go create mode 100644 router/priority/priority_test.go diff --git a/router/priority/BUILD.bazel b/router/priority/BUILD.bazel new file mode 100644 index 0000000000..a5c6bbf1dc --- /dev/null +++ b/router/priority/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library") +load("//tools:go.bzl", "go_test") + +go_library( + name = "go_default_library", + srcs = ["priority.go"], + importpath = "github.com/scionproto/scion/router/priority", + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = [ + "export_test.go", + "priority_test.go", + ], + embed = [":go_default_library"], + deps = ["@com_github_stretchr_testify//assert:go_default_library"], +) diff --git a/router/priority/export_test.go b/router/priority/export_test.go new file mode 100644 index 0000000000..eb79f41c37 --- /dev/null +++ b/router/priority/export_test.go @@ -0,0 +1,26 @@ +// Copyright 2026 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package priority + +func ReadBlockingReflect[T any](queue Queue[T]) (T, bool) { + return readBlockingReflect(queue) +} + +func NewReflectWrapper[T any](queue Queue[T]) *reflectWrapper[T] { + return newReflectWrapper(queue) +} + +func (w *reflectWrapper[T]) ReadBlocking() (T, bool) { + return w.readBlocking() +} diff --git a/router/priority/priority.go b/router/priority/priority.go new file mode 100644 index 0000000000..83b47e2b8f --- /dev/null +++ b/router/priority/priority.go @@ -0,0 +1,111 @@ +// Copyright 2025 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package priority + +import ( + "reflect" +) + +type PriorityLabel int + +const ( + WithPriority PriorityLabel = iota + WithBestEffort + lastPriority + + QueueCount = int(lastPriority) +) + +type Queue[T any] [QueueCount]<-chan T + +// ReadAsync returns a value from the queues read in their priority order. +// If no value is available, this function does not block and returns false. +func ReadAsync[T any](queue Queue[T]) (T, bool) { + var v T + var ok bool +loop: + for _, q := range queue { + select { + case v, ok = <-q: + break loop + default: + } + } + return v, ok +} + +// ReadBlocking returns the first available value from the queues, retrieved in priority order. +// If no value is available at any queue, it blocks until one queue receives a value. +// It returns the value, and a boolean indicating whether all channels are closed. +// XXX(juagargi) In Go, there isn't a general method to synchronously read a value from multiple +// channels. There exists reflect.Select, but it's expensive (see unexported functions below). +// Instead, we +func ReadBlocking[T any](queue Queue[T]) (T, bool) { + // First read in priority order. + v, ok := ReadAsync(queue) + if ok { + return v, ok + } + // Block until any queue has a value. + select { + case v, ok = <-queue[0]: + case v, ok = <-queue[1]: + } + return v, ok +} + +// The following functions are left here only for reference and to test the performance of the +// methods based on reflect logic. They are not exported and not used outside the benchmarks. + +func readBlockingReflect[T any](queue Queue[T]) (T, bool) { + cases := []reflect.SelectCase{} + for i := range queue { + c := reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(queue[i]), + } + cases = append(cases, c) + } + chosenCase, v, ok := reflect.Select(cases) + _ = chosenCase + vv := v.Interface().(T) + return vv, ok +} + +type reflectWrapper[T any] struct { + queue Queue[T] + selectCases []reflect.SelectCase +} + +func newReflectWrapper[T any](queue Queue[T]) *reflectWrapper[T] { + cases := []reflect.SelectCase{} + for i := range queue { + c := reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(queue[i]), + } + cases = append(cases, c) + } + return &reflectWrapper[T]{ + queue: queue, + selectCases: cases, + } +} + +func (w *reflectWrapper[T]) readBlocking() (T, bool) { + chosenCase, v, ok := reflect.Select(w.selectCases) + _ = chosenCase + vv := v.Interface().(T) + return vv, ok +} diff --git a/router/priority/priority_test.go b/router/priority/priority_test.go new file mode 100644 index 0000000000..01a72789dd --- /dev/null +++ b/router/priority/priority_test.go @@ -0,0 +1,191 @@ +// Copyright 2026 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package priority_test + +import ( + "sync/atomic" + "testing" + "time" + + pr "github.com/scionproto/scion/router/priority" + "github.com/stretchr/testify/assert" +) + +func TestReadAsync(t *testing.T) { + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + v, ok := pr.ReadAsync(queues) + assert.Equal(t, false, ok) + assert.Equal(t, 0, v) + + // Send one value to each queue. + queuesOut[1] <- 101 + queuesOut[0] <- 10 + + // Should return the value from queue 0. + v, ok = pr.ReadAsync(queues) + assert.Equal(t, true, ok) + assert.Equal(t, 10, v) + // Should return the value from queue 1. + v, ok = pr.ReadAsync(queues) + assert.Equal(t, true, ok) + assert.Equal(t, 101, v) + + // Should be empty now. + v, ok = pr.ReadAsync(queues) + assert.Equal(t, false, ok) + + // Send to best-effort queue. + queuesOut[1] <- 102 + v, ok = pr.ReadAsync(queues) + assert.Equal(t, true, ok) + assert.Equal(t, 102, v) + + // Should be empty again. + v, ok = pr.ReadAsync(queues) + assert.Equal(t, false, ok) + + // Send to priority queue only. + queuesOut[0] <- 11 + v, ok = pr.ReadAsync(queues) + assert.Equal(t, true, ok) + assert.Equal(t, 11, v) + + // Should be empty again. + v, ok = pr.ReadAsync(queues) + assert.Equal(t, false, ok) +} + +func TestReadBlocking(t *testing.T) { + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + // Send values. + queuesOut[1] <- 100 + queuesOut[0] <- 10 + + // Should not block. + v, ok := pr.ReadBlocking(queues) + assert.Equal(t, true, ok) + assert.Equal(t, 10, v) + v, ok = pr.ReadBlocking(queues) + assert.Equal(t, true, ok) + assert.Equal(t, 100, v) + + // This should block until new values arrive. + finishedRead := atomic.Uint32{} + go func() { + for { + v, ok = pr.ReadBlocking(queues) + finishedRead.Add(1) + } + }() + time.Sleep(10 * time.Millisecond) + assert.Equal(t, uint32(0), finishedRead.Load()) + + // Send to priority. + v, ok = 0, false + queuesOut[0] <- 11 + time.Sleep(10 * time.Millisecond) + assert.Equal(t, uint32(1), finishedRead.Load()) + assert.Equal(t, true, ok) + assert.Equal(t, 11, v) + + // Send to best-effort. + v, ok = 0, false + queuesOut[1] <- 101 + time.Sleep(10 * time.Millisecond) + assert.Equal(t, uint32(2), finishedRead.Load()) + assert.Equal(t, true, ok) + assert.Equal(t, 101, v) + + // No more values. + v, ok = 0, false + time.Sleep(50 * time.Millisecond) + assert.Equal(t, uint32(2), finishedRead.Load()) +} + +func TestReadBlockingReflect(t *testing.T) { + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + + // Send one value to each queue. + queuesOut[1] <- 101 + queuesOut[0] <- 10 + + v, ok := pr.ReadBlockingReflect(queues) + assert.Equal(t, true, ok) + t.Logf("v = %d", v) + v, ok = pr.ReadBlockingReflect(queues) + assert.Equal(t, true, ok) + t.Logf("v = %d", v) +} + +func BenchmarkReadBlocking(b *testing.B) { + b.Run("readblocking", func(b *testing.B) { + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + go func() { + for { + queuesOut[0] <- 0 + queuesOut[1] <- 1 + } + }() + for range b.N { + pr.ReadBlocking(queues) + } + }) + b.Run("reflect", func(b *testing.B) { + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + go func() { + for { + queuesOut[0] <- 0 + queuesOut[1] <- 1 + } + }() + for range b.N { + pr.ReadBlockingReflect(queues) + } + }) + b.Run("reflect-wrapper", func(b *testing.B) { + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + wrapper := pr.NewReflectWrapper(queues) + go func() { + for { + queuesOut[0] <- 0 + queuesOut[1] <- 1 + } + }() + for range b.N { + wrapper.ReadBlocking() + } + }) +} + +func newPriorityQueue(channelsBufferSize int) [pr.QueueCount]chan int { + var q [pr.QueueCount]chan int + for i := range q { + q[i] = make(chan int, channelsBufferSize) + } + return q +} + +func toInChannels(q [pr.QueueCount]chan int) [pr.QueueCount]<-chan int { + var ret [pr.QueueCount]<-chan int + for i := range pr.QueueCount { + ret[i] = q[i] + } + return ret +} From 6a65a7806d816de4a20ecfa689c025e39e713103 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Fri, 9 Jan 2026 10:16:21 +0100 Subject: [PATCH 03/16] Use multiple queues (with priorities) at udp/ip underlay provider. The udp/ip underlay provider now uses priority queues to send the received packets. This is done by reading first all of the priority queue, and only then reading the best-effort one. Also, modify the layout of the Packet struct to automatically align to 64 bytes, instead of manually adding padding (which failed for zero padding bytes). --- router/BUILD.bazel | 1 + router/dataplane.go | 52 +++++++++++-- router/priority/priority.go | 2 +- router/underlayproviders/udpip/BUILD.bazel | 3 +- router/underlayproviders/udpip/udpip.go | 89 +++++++++++++++------- 5 files changed, 108 insertions(+), 39 deletions(-) diff --git a/router/BUILD.bazel b/router/BUILD.bazel index 9d42f1a9da..1d4ec0afa3 100644 --- a/router/BUILD.bazel +++ b/router/BUILD.bazel @@ -38,6 +38,7 @@ go_library( "//router/bfd:go_default_library", "//router/config:go_default_library", "//router/control:go_default_library", + "//router/priority:go_default_library", "@com_github_gopacket_gopacket//:go_default_library", "@com_github_gopacket_gopacket//layers:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library", diff --git a/router/dataplane.go b/router/dataplane.go index 3e8466d3cb..cfefc2d69f 100644 --- a/router/dataplane.go +++ b/router/dataplane.go @@ -53,6 +53,7 @@ import ( underlayconn "github.com/scionproto/scion/private/underlay/conn" "github.com/scionproto/scion/router/bfd" "github.com/scionproto/scion/router/control" + pr "github.com/scionproto/scion/router/priority" ) const ( @@ -120,6 +121,17 @@ const ( // arch) until SlowpathRequest which is 4 bytes long. The rest is in decreasing order of size and // size-aligned. We want to fit neatly into cache lines, so we need to fit in 64 bytes. The padding // required to occupy exactly 64 bytes depends on the architecture. +// +// Note(juagargi): if the Packet struct grows larger than 64 bytes, it should have a size multiple +// of 64 bytes. This prevents "false sharing", i.e. having the same bytes being accessed by +// multiple threads simultaneously, thus failing cache coherence and hurting performance. +// This "Packet struct alignment to 64 bytes" is achieved through the presence of the field +// `_ [_pad]byte`. The value `_pad` is computed via a helper struct `alignHelperForPacket`, +// who contains the same exact fields and in the same order as in Packet. +// The presence of the _ [_pad]field needs to be not at the last position of the struct for there +// is a specific case (with _pad==0) where the compiler would add extra padding to avoid pointer +// aliasing with the next Packet object. The last field in the structure (QueueIndex PriorityLabel) +// must not introduce additional padding due to its alignment. type Packet struct { // The useful part of the raw packet at a point in time (i.e. a slice of the full buffer). It // can be any portion of the full buffer; not necessarily the start. This code maintains the @@ -140,10 +152,40 @@ type Packet struct { // The type of traffic. This is used for metrics at the forwarding stage, but is most // economically determined at the processing stage. So store it here. It's 2 bytes long. trafficType trafficType - // Pad to 64 bytes. For 64bit arch, add 1 byte. For 32bit arch, add 29 bytes. - _ [1 + is32bit*28]byte + // The struct padding field cannot be the last field of the struct. This is because if the + // helper constant _pad is zero and the field is at the end, the compiler will need to avoid + // aliasing this field with the next struct's pointer (e.g. in an array). + // Since the real last field of this struct is a byte long, this does't introduce alignment + // issues (and thus does not modify the final size of the struct regardless of the value + // of _pad). See notes for the Packet struct. + _ [_pad]byte + // Priority forwarding label: packets with more priority are forwarded first + QueueIndex pr.PriorityLabel +} + +// alignHelperForPacket is only used to compute the initial size of the Packet struct without +// any extra padding. Since we can't define Packet recursively in terms of Packet without padding, +// an extra struct is necessary. +// The alignHelperForPacket fields must be kept in synchrony with Packet. +type alignHelperForPacket struct { + RawPacket []byte + buffer *[bufSize]byte + RemoteAddr unsafe.Pointer + Link Link + slowPathRequest slowPathRequest + egress uint16 + trafficType trafficType + QueueIndex pr.PriorityLabel } +// Make sure that the packet structure has the size we expect. +const ( + _pad = (64 - int(unsafe.Sizeof(alignHelperForPacket{})%64)) % 64 +) + +// Fail (negative array size) if the struct is not a multiple of 64. +var _ [-(int(unsafe.Sizeof(Packet{}) % 64))]byte + // Keep this 4 bytes long. See comment for packet. type slowPathRequest struct { pointer uint16 @@ -151,12 +193,6 @@ type slowPathRequest struct { code slayers.SCMPCode } -// Make sure that the packet structure has the size we expect. -const ( - _ uintptr = 64 - unsafe.Sizeof(Packet{}) // assert 64 >= sizeof(Packet) - _ uintptr = unsafe.Sizeof(Packet{}) - 64 // assert sizeof(Packet) >= 64 -) - // initPacket configures the given blank packet (and returns it, for convenience). func (p *Packet) init(buffer *[bufSize]byte) *Packet { p.buffer = buffer diff --git a/router/priority/priority.go b/router/priority/priority.go index 83b47e2b8f..20c20aeae0 100644 --- a/router/priority/priority.go +++ b/router/priority/priority.go @@ -17,7 +17,7 @@ import ( "reflect" ) -type PriorityLabel int +type PriorityLabel uint8 const ( WithPriority PriorityLabel = iota diff --git a/router/underlayproviders/udpip/BUILD.bazel b/router/underlayproviders/udpip/BUILD.bazel index 3437619762..3c308521a9 100644 --- a/router/underlayproviders/udpip/BUILD.bazel +++ b/router/underlayproviders/udpip/BUILD.bazel @@ -14,11 +14,12 @@ go_library( "//pkg/log:go_default_library", "//pkg/private/serrors:go_default_library", "//pkg/slayers:go_default_library", - "//pkg/stun:go_default_library", "//pkg/slices:go_default_library", + "//pkg/stun:go_default_library", "//private/underlay/conn:go_default_library", "//router:go_default_library", "//router/bfd:go_default_library", + "//router/priority:go_default_library", ], ) diff --git a/router/underlayproviders/udpip/udpip.go b/router/underlayproviders/udpip/udpip.go index 960e008663..7ecbe7d08f 100644 --- a/router/underlayproviders/udpip/udpip.go +++ b/router/underlayproviders/udpip/udpip.go @@ -36,6 +36,7 @@ import ( "github.com/scionproto/scion/private/underlay/conn" "github.com/scionproto/scion/router" "github.com/scionproto/scion/router/bfd" + pr "github.com/scionproto/scion/router/priority" ) var ( @@ -205,10 +206,10 @@ func (u *provider) Stop() { // if sibling links are to have distinct connections). type udpConnection struct { conn router.BatchConn - name string // for logs. It's more informative than ifID. - link udpLink // Link with exclusive use of the connection. - links map[netip.AddrPort]udpLink // Links that share this connection - queue chan *router.Packet // Packets to be sent. + name string // for logs. It's more informative than ifID. + link udpLink // Link with exclusive use of the connection. + links map[netip.AddrPort]udpLink // Links that share this connection + queues [pr.QueueCount]chan *router.Packet // Packets to be sent, with priorities. metrics *router.InterfaceMetrics receiverDone chan struct{} senderDone chan struct{} @@ -247,13 +248,19 @@ func (u *udpConnection) stop() { wasRunning := u.running.Swap(false) if wasRunning { - u.conn.Close() // Unblock receiver - close(u.queue) // Unblock sender + u.conn.Close() // Unblock receiver + u.closeQueues() // Unblock sender <-u.receiverDone <-u.senderDone } } +func (u *udpConnection) closeQueues() { + for _, q := range u.queues { + close(q) + } +} + func (u *udpConnection) receive(batchSize int, pool router.PacketPool) { log.Debug("Receive", "connection", u.name) @@ -318,9 +325,11 @@ func (u *udpConnection) receive(batchSize int, pool router.PacketPool) { func readUpTo( pktIter sslices.Iter[**router.Packet], // Where to write the packet pointer. - queue <-chan *router.Packet, // Packet pointer source. + queues [pr.QueueCount]chan *router.Packet, // Packet pointer source. needsBlocking bool, ) int { + inQueues := typeCastIngressQueues(queues) + // This is the reading function pointer. var read func(**router.Packet) bool @@ -330,16 +339,13 @@ func readUpTo( read = func(ptr **router.Packet) bool { readBlock := func(ptr **router.Packet) bool { var ok bool - *ptr, ok = <-queue + *ptr, ok = pr.ReadBlocking(inQueues) return ok } readAsync := func(ptr **router.Packet) bool { var ok bool - select { - case *ptr, ok = <-queue: - default: - } + *ptr, ok = pr.ReadAsync(inQueues) return ok } @@ -391,7 +397,7 @@ func (u *udpConnection) send(batchSize int, pool router.PacketPool) { // no more than the count of free buckets. newBatchPktCount := readUpTo( sslices.CDIterator(pkts, currentIdx+toWrite, batchSize-toWrite), - u.queue, + u.queues, toWrite == 0) // Turn the packets into underlay messages that WriteBatch can send. @@ -468,7 +474,7 @@ func makeHashSeed() uint32 { type connectedLink struct { procQs []chan *router.Packet name string // For logs - egressQ chan<- *router.Packet + egressQs [pr.QueueCount]chan<- *router.Packet metrics *router.InterfaceMetrics pool router.PacketPool bfdSession *bfd.Session @@ -522,22 +528,23 @@ func (u *provider) newConnectedLink( if err != nil { return nil, err } - queue := make(chan *router.Packet, qSize) + queues := createQueues(qSize) el := &connectedLink{ name: remoteAddr.String(), - egressQ: queue, + egressQs: typeCastEgressQueues(queues), metrics: metrics, bfdSession: bfd, seed: makeHashSeed(), ifID: ifID, scope: scope, } + c := &udpConnection{ conn: conn, name: el.name, link: el, // links: nil; no demux lookup ever for this connection - queue: queue, + queues: queues, metrics: metrics, // send() needs them :-( receiverDone: make(chan struct{}), senderDone: make(chan struct{}), @@ -548,6 +555,30 @@ func (u *provider) newConnectedLink( return el, nil } +func createQueues(qSize int) [pr.QueueCount]chan *router.Packet { + var queues [pr.QueueCount]chan *router.Packet + for i := range queues { + queues[i] = make(chan *router.Packet, qSize) + } + return queues +} + +func typeCastEgressQueues(queues [pr.QueueCount]chan *router.Packet) [pr.QueueCount]chan<- *router.Packet { + var ret [pr.QueueCount]chan<- *router.Packet + for i := range queues { + ret[i] = queues[i] + } + return ret +} + +func typeCastIngressQueues(queues [pr.QueueCount]chan *router.Packet) [pr.QueueCount]<-chan *router.Packet { + var ret [pr.QueueCount]<-chan *router.Packet + for i := range queues { + ret[i] = queues[i] + } + return ret +} + func (l *connectedLink) start( ctx context.Context, procQs []chan *router.Packet, @@ -602,7 +633,7 @@ func (l *connectedLink) Resolve(p *router.Packet, host addr.Host, port uint16) e func (l *connectedLink) Send(p *router.Packet) bool { select { - case l.egressQ <- p: + case l.egressQs[p.QueueIndex] <- p: default: return false } @@ -611,7 +642,7 @@ func (l *connectedLink) Send(p *router.Packet) bool { func (l *connectedLink) SendBlocking(p *router.Packet) { // We use a bound and connected socket so we don't need to specify the destination. - l.egressQ <- p + l.egressQs[p.QueueIndex] <- p } func (l *connectedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet) { @@ -644,7 +675,7 @@ func (l *connectedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet type detachedLink struct { procQs []chan *router.Packet name string // For logs - egressQ chan<- *router.Packet + egressQs [pr.QueueCount]chan<- *router.Packet metrics *router.InterfaceMetrics pool router.PacketPool bfdSession *bfd.Session @@ -706,7 +737,7 @@ func (u *provider) newDetachedLink( sl := &detachedLink{ name: remoteAddr.String(), - egressQ: c.queue, + egressQs: typeCastEgressQueues(c.queues), metrics: metrics, bfdSession: bfd, remote: net.UDPAddrFromAddrPort(remoteAddr), @@ -776,7 +807,7 @@ func (l *detachedLink) Send(p *router.Packet) bool { // is safe because we treat p.RemoteAddr as immutable and the router main code doesn't touch it. p.RemoteAddr = unsafe.Pointer(l.remote) select { - case l.egressQ <- p: + case l.egressQs[p.QueueIndex] <- p: default: return false } @@ -786,7 +817,7 @@ func (l *detachedLink) Send(p *router.Packet) bool { func (l *detachedLink) SendBlocking(p *router.Packet) { // Same as Send(). We must supply the destination address. p.RemoteAddr = unsafe.Pointer(l.remote) - l.egressQ <- p + l.egressQs[p.QueueIndex] <- p } func (l *detachedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet) { @@ -816,9 +847,9 @@ func (l *detachedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet) type internalLink struct { procQ chan *router.Packet procQs []chan *router.Packet - egressQ chan *router.Packet procStop chan struct{} procDone chan struct{} + egressQs [pr.QueueCount]chan<- *router.Packet metrics *router.InterfaceMetrics pool router.PacketPool svc *router.Services[netip.AddrPort] @@ -853,9 +884,9 @@ func (u *provider) NewInternalLink( return nil, err } u.internalHashSeed = makeHashSeed() - queue := make(chan *router.Packet, qSize) + queues := createQueues(qSize) il := &internalLink{ - egressQ: queue, + egressQs: typeCastEgressQueues(queues), metrics: metrics, svc: u.svc, seed: u.internalHashSeed, @@ -868,7 +899,7 @@ func (u *provider) NewInternalLink( name: "internal", link: il, // links: see below. - queue: queue, + queues: queues, metrics: metrics, // send() needs them :-( receiverDone: make(chan struct{}), senderDone: make(chan struct{}), @@ -1042,7 +1073,7 @@ func (l *internalLink) Resolve(p *router.Packet, dst addr.Host, port uint16) err // The packet's destination is already in the packet's meta-data. func (l *internalLink) Send(p *router.Packet) bool { select { - case l.egressQ <- p: + case l.egressQs[p.QueueIndex] <- p: default: return false } @@ -1051,7 +1082,7 @@ func (l *internalLink) Send(p *router.Packet) bool { // The packet's destination is already in the packet's meta-data. func (l *internalLink) SendBlocking(p *router.Packet) { - l.egressQ <- p + l.egressQs[p.QueueIndex] <- p } func (l *internalLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet) { From 444cf08bbbf7efeab3e86a8f6be376c388156a9e Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Mon, 12 Jan 2026 10:10:38 +0100 Subject: [PATCH 04/16] Fix performance bug in priority queue. --- router/priority/priority.go | 18 +++- router/priority/priority_test.go | 140 +++++++++++++++++++++++++------ 2 files changed, 128 insertions(+), 30 deletions(-) diff --git a/router/priority/priority.go b/router/priority/priority.go index 20c20aeae0..32a3af6992 100644 --- a/router/priority/priority.go +++ b/router/priority/priority.go @@ -38,6 +38,10 @@ loop: for _, q := range queue { select { case v, ok = <-q: + if !ok { + // Channel is closed. + continue + } break loop default: } @@ -52,17 +56,23 @@ loop: // channels. There exists reflect.Select, but it's expensive (see unexported functions below). // Instead, we func ReadBlocking[T any](queue Queue[T]) (T, bool) { - // First read in priority order. + // Compile guards because we have manual code below that is written only for the + // case of two queues (channels) in the Queue definition. + var _ [2 - len(queue)]int // assert( len(queue) <= 2 ) + var _ [len(queue) - 2]int // assert( len(queue) >= 2 ) + v, ok := ReadAsync(queue) if ok { + // We got a value we can return. return v, ok } // Block until any queue has a value. select { - case v, ok = <-queue[0]: - case v, ok = <-queue[1]: + case v, ok := <-queue[0]: + return v, ok + case v, ok := <-queue[1]: + return v, ok } - return v, ok } // The following functions are left here only for reference and to test the performance of the diff --git a/router/priority/priority_test.go b/router/priority/priority_test.go index 01a72789dd..0d018cd8fb 100644 --- a/router/priority/priority_test.go +++ b/router/priority/priority_test.go @@ -131,45 +131,133 @@ func TestReadBlockingReflect(t *testing.T) { t.Logf("v = %d", v) } -func BenchmarkReadBlocking(b *testing.B) { - b.Run("readblocking", func(b *testing.B) { - queuesOut := newPriorityQueue(2) - queues := toInChannels(queuesOut) +// BenchmarkRead measures how long it takes to obtain N "values" from the queues, +// using different methods. +func BenchmarkRead(b *testing.B) { + const N = 1024 + + b.Run("blocking-native", func(b *testing.B) { + for range b.N { + queuesOut := [2]chan int{ + make(chan int), + make(chan int), + } + queues := toInChannels(queuesOut) + go func() { + for { + queuesOut[0] <- 0 + queuesOut[1] <- 1 + } + }() + for range N { + // Try from queue 0: + select { + case <-queues[0]: + default: + // Then try from queue 1: + select { + case <-queues[1]: + default: + // Both empty, try both without ordering: + select { + case <-queues[0]: + case <-queues[1]: + } + } + } + } + } + }) + + b.Run("async-native", func(b *testing.B) { + queues := [2]chan int{ + make(chan int), + make(chan int), + } + queuesIn := toInChannels(queues) go func() { for { - queuesOut[0] <- 0 - queuesOut[1] <- 1 + queues[0] <- 0 + queues[1] <- 1 } }() for range b.N { - pr.ReadBlocking(queues) + for range b.N { + // read one value: + var ok bool + for !ok { + select { + case _, ok = <-queuesIn[0]: + case _, ok = <-queuesIn[1]: + default: + } + } + } } }) - b.Run("reflect", func(b *testing.B) { - queuesOut := newPriorityQueue(2) - queues := toInChannels(queuesOut) - go func() { - for { - queuesOut[0] <- 0 - queuesOut[1] <- 1 + + b.Run("blocking", func(b *testing.B) { + for range b.N { + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + go func() { + for { + queuesOut[0] <- 0 + queuesOut[1] <- 1 + } + }() + for range N { + pr.ReadBlocking(queues) } - }() + } + }) + b.Run("async", func(b *testing.B) { for range b.N { - pr.ReadBlockingReflect(queues) + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + go func() { + for { + queuesOut[0] <- 0 + queuesOut[1] <- 1 + } + }() + for range N { + var ok bool + for !ok { + _, ok = pr.ReadAsync(queues) + } + } } }) - b.Run("reflect-wrapper", func(b *testing.B) { - queuesOut := newPriorityQueue(2) - queues := toInChannels(queuesOut) - wrapper := pr.NewReflectWrapper(queues) - go func() { - for { - queuesOut[0] <- 0 - queuesOut[1] <- 1 + b.Run("blocking-reflect", func(b *testing.B) { + for range b.N { + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + go func() { + for { + queuesOut[0] <- 0 + queuesOut[1] <- 1 + } + }() + for range N { + pr.ReadBlockingReflect(queues) } - }() + } + }) + b.Run("blocking-wrapper", func(b *testing.B) { for range b.N { - wrapper.ReadBlocking() + queuesOut := newPriorityQueue(2) + queues := toInChannels(queuesOut) + wrapper := pr.NewReflectWrapper(queues) + go func() { + for { + queuesOut[0] <- 0 + queuesOut[1] <- 1 + } + }() + for range N { + wrapper.ReadBlocking() + } } }) } From d27a39efc563574b64a019583a19b82635c2b18f Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Mon, 12 Jan 2026 10:33:26 +0100 Subject: [PATCH 05/16] Added tests and benchmarks comparing old behavior, via testhooks. --- router/dataplane_testhooks.go | 43 +++ .../udpip/perf_comparisons_test.go | 356 ++++++++++++++++++ 2 files changed, 399 insertions(+) create mode 100644 router/dataplane_testhooks.go create mode 100644 router/underlayproviders/udpip/perf_comparisons_test.go diff --git a/router/dataplane_testhooks.go b/router/dataplane_testhooks.go new file mode 100644 index 0000000000..e4dbd0828d --- /dev/null +++ b/router/dataplane_testhooks.go @@ -0,0 +1,43 @@ +// Copyright 2026 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build testhooks + +// This file is built only with the "testhooks" build tag. It is intended for tests to +// efficiently access some internals of the router package. +// It should not be used otherwise. + +package router + +const ( + TtMax = ttMax + TtOther = ttOther + MinSizeClass = minSizeClass + MaxSizeClass = maxSizeClass +) + +func MakePacketPool(poolSize, headroom int) PacketPool { + pool := makePacketPool(poolSize, headroom) + + pktBuffers := make([][bufSize]byte, poolSize) + pktStructs := make([]Packet, poolSize) + for i := 0; i < poolSize; i++ { + pool.Put(pktStructs[i].init(&pktBuffers[i])) + } + return pool +} + +func (p Packet) GetTrafficType() trafficType { + return p.trafficType +} diff --git a/router/underlayproviders/udpip/perf_comparisons_test.go b/router/underlayproviders/udpip/perf_comparisons_test.go new file mode 100644 index 0000000000..b402357c7e --- /dev/null +++ b/router/underlayproviders/udpip/perf_comparisons_test.go @@ -0,0 +1,356 @@ +// Copyright 2026 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build testhooks + +// These tests and benchmarks rely on accessing the internals of the router package. +// To run them, the build tag "testhooks" must be provided. E.g. +// go test -tags=testhooks ./router/underlayproviders/udpip/ + +package udpip + +import ( + "net" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/prometheus/client_golang/prometheus" + "github.com/scionproto/scion/pkg/log" + + "github.com/scionproto/scion/pkg/slices" + "github.com/scionproto/scion/private/underlay/conn" + underlayconn "github.com/scionproto/scion/private/underlay/conn" + "github.com/scionproto/scion/router" + "github.com/scionproto/scion/router/mock_router" +) + +func TestReadUpTo(t *testing.T) { + const batchSize = 256 + const N = 1024 * 1024 + t.Run("new", func(t *testing.T) { + queues := createQueues(batchSize) + go func() { + for range N { + queues[1] <- &router.Packet{} + } + close(queues[0]) + close(queues[1]) + }() + pkts := make([]*router.Packet, batchSize) + iter := slices.CDIterator(pkts, 0, len(pkts)) + for rem := N; rem > 0; { + read := readUpTo(iter, queues, true) + rem -= read + } + }) + t.Run("old", func(t *testing.T) { + ch := make(chan *router.Packet, batchSize) + go func() { + for range N { + ch <- &router.Packet{} + } + close(ch) + }() + pkts := make([]*router.Packet, batchSize) + + for rem := N; rem > 0; { + read := oldReadUpTo(ch, batchSize, true, pkts) + rem -= read + } + }) +} + +func TestSend(t *testing.T) { + const PacketsToSendCount = 1024 * 1024 + + t.Run("new", func(t *testing.T) { + u := createUdpConnection(t, 256) + pool := router.MakePacketPool(256, 0) + + // Send some packets. + go func() { + for range PacketsToSendCount { + pkt := pool.Get() + u.queues[1] <- pkt + } + // Stop the forwarder. + u.running.Store(false) + for _, q := range u.queues { + close(q) + } + u.conn.Close() + }() + + // Run the forwarding process. + u.send(256, pool) + }) + + t.Run("old", func(t *testing.T) { + u := createUdpConnection(t, 256) + pool := router.MakePacketPool(256, 0) + + // Send some packets. + go func() { + for range PacketsToSendCount { + pkt := pool.Get() + u.queues[1] <- pkt + } + // Stop the forwarder. + u.running.Store(false) + for _, q := range u.queues { + close(q) + } + u.conn.Close() + }() + + // Run the forwarding process. + u.oldSend(256, pool) + }) +} + +func BenchmarkReadUpTo(b *testing.B) { + const batchSize = 256 + const N = 1024 + + b.Run("old", func(b *testing.B) { + for range b.N { + ch := make(chan *router.Packet, batchSize) + go func() { + for range N { + ch <- &router.Packet{} + } + close(ch) + }() + pkts := make([]*router.Packet, batchSize) + for rem := N; rem > 0; { + read := oldReadUpTo(ch, batchSize, true, pkts) + rem -= read + } + } + }) + + b.Run("new", func(b *testing.B) { + for range b.N { + queues := createQueues(batchSize) + go func() { + for range N { + queues[1] <- &router.Packet{} + } + close(queues[0]) + close(queues[1]) + }() + pkts := make([]*router.Packet, batchSize) + iter := slices.CDIterator(pkts, 0, len(pkts)) + for rem := N; rem > 0; { + read := readUpTo(iter, queues, true) + rem -= read + } + } + }) + +} + +// cpu: Intel(R) Core(TM) i7-7700T CPU @ 2.90GHz +// BenchmarkSend/old-8 5379 283566 ns/op +// BenchmarkSend/new-8 3517 367717 ns/op +func BenchmarkSend(b *testing.B) { + const batchSize = 256 + const PacketsToSend = 1024 + + b.Run("old", func(b *testing.B) { + for range b.N { + b.StopTimer() + u := createUdpConnection(b, batchSize) + pool := router.MakePacketPool(batchSize, 0) + + // Send some packets. + go func() { + for range PacketsToSend { + pkt := pool.Get() + u.queues[1] <- pkt + } + // Stop the forwarder. + u.running.Store(false) + for _, q := range u.queues { + close(q) + } + u.conn.Close() + }() + b.StartTimer() + u.oldSend(256, pool) + } + }) + + b.Run("new", func(b *testing.B) { + for range b.N { + b.StopTimer() + u := createUdpConnection(b, batchSize) + pool := router.MakePacketPool(batchSize, 0) + + // Send some packets. + go func() { + for range PacketsToSend { + pkt := pool.Get() + u.queues[1] <- pkt + } + // Stop the forwarder. + u.running.Store(false) + for _, q := range u.queues { + close(q) + } + u.conn.Close() + }() + b.StartTimer() + u.send(256, pool) + } + }) +} + +func oldReadUpTo(queue <-chan *router.Packet, n int, needsBlocking bool, pkts []*router.Packet) int { + i := 0 + if needsBlocking { + p, ok := <-queue + if !ok { + return i + } + pkts[i] = p + i++ + } + + for ; i < n; i++ { + select { + case p, ok := <-queue: + if !ok { + return i + } + pkts[i] = p + default: + return i + } + } + return i +} + +func (u *udpConnection) oldSend(batchSize int, pool router.PacketPool) { + log.Debug("Send", "connection", u.name) + + // We use this somewhat like a ring buffer. + pkts := make([]*router.Packet, batchSize) + + // We use this as a temporary buffer, but allocate it just once + // to save on garbage handling. + msgs := make(conn.Messages, batchSize) + for i := range msgs { + msgs[i].Buffers = make([][]byte, 1) + } + + queue := u.queues[1] + conn := u.conn + metrics := u.metrics + toWrite := 0 + + for u.running.Load() { + // Top-up our batch. + toWrite += oldReadUpTo(queue, batchSize-toWrite, toWrite == 0, pkts[toWrite:]) + + // Turn the packets into underlay messages that WriteBatch can send. + for i, p := range pkts[:toWrite] { + msgs[i].Buffers[0] = p.RawPacket + msgs[i].Addr = nil + // If we're using a connected socket we must not specify the address. It might cause + // redundant route queries and the address might not even be set in the packet. + // Otherwise, we must specify the address. + if !u.connected { + msgs[i].Addr = (*net.UDPAddr)(p.RemoteAddr) + } + } + + written, _ := conn.WriteBatch(msgs[:toWrite], 0) + if written < 0 { + // WriteBatch returns -1 on error, we just consider this as + // 0 packets written + written = 0 + } + oldUpdateOutputMetrics(metrics, pkts[:written]) + for _, p := range pkts[:written] { + pool.Put(p) + } + if written != toWrite { + // Only one is dropped at this time. We'll retry the rest. + sc := router.ClassOfSize(len(pkts[written].RawPacket)) + metrics[sc].DroppedPacketsInvalid.Inc() + pool.Put(pkts[written]) + toWrite -= (written + 1) + // Shift the leftovers to the head of the buffers. + for i := 0; i < toWrite; i++ { + pkts[i] = pkts[i+written+1] + } + } else { + toWrite = 0 + } + } +} + +func oldUpdateOutputMetrics(metrics *router.InterfaceMetrics, packets []*router.Packet) { + // We need to collect stats by traffic type and size class. + // Try to reduce the metrics lookup penalty by using some + // simpler staging data structure. + writtenPkts := [router.TtMax][router.MaxSizeClass]int{} + writtenBytes := [router.TtMax][router.MaxSizeClass]int{} + for _, p := range packets { + s := len(p.RawPacket) + sc := router.ClassOfSize(s) + tt := p.GetTrafficType() + writtenPkts[tt][sc]++ + writtenBytes[tt][sc] += s + } + for t := router.TtOther; t < router.TtMax; t++ { + for sc := router.MinSizeClass; sc < router.MaxSizeClass; sc++ { + if writtenPkts[t][sc] > 0 { + metrics[sc].Output[t].OutputPacketsTotal.Add(float64(writtenPkts[t][sc])) + metrics[sc].Output[t].OutputBytesTotal.Add(float64(writtenBytes[t][sc])) + } + } + } +} + +func createUdpConnection(t gomock.TestReporter, queueSize int) *udpConnection { + ctrl := gomock.NewController(t) + mConn := mock_router.NewMockBatchConn(ctrl) + mConn.EXPECT().WriteBatch(gomock.Any(), 0).AnyTimes().DoAndReturn( + func(msgs underlayconn.Messages, flags int) (int, error) { + // fmt.Printf("sent %d packets\n", len(msgs)) + time.Sleep(time.Duration(len(msgs)) * time.Nanosecond) + return len(msgs), nil + }) + mConn.EXPECT().Close().AnyTimes().Return(nil) + + metrics := &router.InterfaceMetrics{} + noOpts := prometheus.CounterOpts{} + for i := range metrics { + metrics[i].DroppedPacketsInvalid = prometheus.NewCounter(noOpts) + for j := range 6 { + metrics[i].Output[j].OutputBytesTotal = prometheus.NewCounter(noOpts) + metrics[i].Output[j].OutputPacketsTotal = prometheus.NewCounter(noOpts) + } + } + u := &udpConnection{ + queues: createQueues(queueSize), + conn: mConn, + metrics: metrics, + } + u.running.Store(true) + return u +} From 61a5524a39c4032ffabca146b9516bdcbae2388d Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Tue, 13 Jan 2026 15:08:45 +0100 Subject: [PATCH 06/16] BFD packets are treated with priority. --- router/dataplane.go | 3 +++ router/underlayproviders/udpip/udpip.go | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/router/dataplane.go b/router/dataplane.go index cfefc2d69f..6c1929ebbe 100644 --- a/router/dataplane.go +++ b/router/dataplane.go @@ -2217,6 +2217,9 @@ func (b *bfdSend) Send(bfd *layers.BFD) error { // the forwarding queue is an serious internal error. Let that panic. fwLink := b.dataPlane.interfaces[b.ifID] + // BFD packets are always marked as priority. + p.QueueIndex = pr.WithPriority + if !fwLink.Send(p) { // We do not care if some BFD packets get bounced under high load. If it becomes a problem, // the solution is do use BFD's demand-mode. To be considered in a future refactoring. diff --git a/router/underlayproviders/udpip/udpip.go b/router/underlayproviders/udpip/udpip.go index 7ecbe7d08f..52aa4adcce 100644 --- a/router/underlayproviders/udpip/udpip.go +++ b/router/underlayproviders/udpip/udpip.go @@ -393,6 +393,21 @@ func (u *udpConnection) send(batchSize int, pool router.PacketPool) { currentIdx := 0 // Index of the first packet pending to be sent. toWrite := 0 // Amount of packets pending to be sent. for u.running.Load() { + + // XXX(juagargi): open question: if the priority input queue is empty, how many best-effort + // packets should we read and then send? Two answers (to show my hesitation): + // 1. If too many, then while we are sending them new priority packets could arrive, and we + // would be adding latency for those priority packets to be read and sent. + // 2. If too little, then the forwarding process will not be very efficient. + // + // While the solution may affect jitter (or latency in general), it will not affect the + // reliability of the forwarding for priority packets: because we assume that the + // system is well configured, we don't have a higher rate of priority packets reception + // than emission; we assume this even taking into account the tolerable token buckets burst. + // In that case, unless batchSize was configured extremely high, we will not enqueue + // enough priority packets (without sending) them that would cause a bottleneck enough to + // stall the packet processors. + // Top-up our batch. Write onto the ring buffer, starting from the first free "bucket" and // no more than the count of free buckets. newBatchPktCount := readUpTo( From ab293d6631fb3b6009f1ae7d4fe554e3842120b3 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Thu, 22 Jan 2026 09:03:03 +0100 Subject: [PATCH 07/16] Rename Packet.QueueIndex to PriorityLabel. --- router/dataplane.go | 4 ++-- router/underlayproviders/udpip/udpip.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/router/dataplane.go b/router/dataplane.go index 6c1929ebbe..576171a48c 100644 --- a/router/dataplane.go +++ b/router/dataplane.go @@ -160,7 +160,7 @@ type Packet struct { // of _pad). See notes for the Packet struct. _ [_pad]byte // Priority forwarding label: packets with more priority are forwarded first - QueueIndex pr.PriorityLabel + PriorityLabel pr.PriorityLabel } // alignHelperForPacket is only used to compute the initial size of the Packet struct without @@ -2218,7 +2218,7 @@ func (b *bfdSend) Send(bfd *layers.BFD) error { fwLink := b.dataPlane.interfaces[b.ifID] // BFD packets are always marked as priority. - p.QueueIndex = pr.WithPriority + p.PriorityLabel = pr.WithPriority if !fwLink.Send(p) { // We do not care if some BFD packets get bounced under high load. If it becomes a problem, diff --git a/router/underlayproviders/udpip/udpip.go b/router/underlayproviders/udpip/udpip.go index 52aa4adcce..e5cef40ec3 100644 --- a/router/underlayproviders/udpip/udpip.go +++ b/router/underlayproviders/udpip/udpip.go @@ -648,7 +648,7 @@ func (l *connectedLink) Resolve(p *router.Packet, host addr.Host, port uint16) e func (l *connectedLink) Send(p *router.Packet) bool { select { - case l.egressQs[p.QueueIndex] <- p: + case l.egressQs[p.PriorityLabel] <- p: default: return false } @@ -657,7 +657,7 @@ func (l *connectedLink) Send(p *router.Packet) bool { func (l *connectedLink) SendBlocking(p *router.Packet) { // We use a bound and connected socket so we don't need to specify the destination. - l.egressQs[p.QueueIndex] <- p + l.egressQs[p.PriorityLabel] <- p } func (l *connectedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet) { @@ -822,7 +822,7 @@ func (l *detachedLink) Send(p *router.Packet) bool { // is safe because we treat p.RemoteAddr as immutable and the router main code doesn't touch it. p.RemoteAddr = unsafe.Pointer(l.remote) select { - case l.egressQs[p.QueueIndex] <- p: + case l.egressQs[p.PriorityLabel] <- p: default: return false } @@ -832,7 +832,7 @@ func (l *detachedLink) Send(p *router.Packet) bool { func (l *detachedLink) SendBlocking(p *router.Packet) { // Same as Send(). We must supply the destination address. p.RemoteAddr = unsafe.Pointer(l.remote) - l.egressQs[p.QueueIndex] <- p + l.egressQs[p.PriorityLabel] <- p } func (l *detachedLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet) { @@ -1088,7 +1088,7 @@ func (l *internalLink) Resolve(p *router.Packet, dst addr.Host, port uint16) err // The packet's destination is already in the packet's meta-data. func (l *internalLink) Send(p *router.Packet) bool { select { - case l.egressQs[p.QueueIndex] <- p: + case l.egressQs[p.PriorityLabel] <- p: default: return false } @@ -1097,7 +1097,7 @@ func (l *internalLink) Send(p *router.Packet) bool { // The packet's destination is already in the packet's meta-data. func (l *internalLink) SendBlocking(p *router.Packet) { - l.egressQs[p.QueueIndex] <- p + l.egressQs[p.PriorityLabel] <- p } func (l *internalLink) receive(size int, srcAddr *net.UDPAddr, p *router.Packet) { From 9350a8a027dbca2c35bd34c73e658f3da0d74ed0 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Thu, 22 Jan 2026 09:03:41 +0100 Subject: [PATCH 08/16] Reset the packet priority label when getting the packet from the pool. --- router/dataplane.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/router/dataplane.go b/router/dataplane.go index 576171a48c..8a7fd48945 100644 --- a/router/dataplane.go +++ b/router/dataplane.go @@ -204,8 +204,9 @@ func (p *Packet) init(buffer *[bufSize]byte) *Packet { // relative to the buffer, so there's enough headroom for any underlay headers. func (p *Packet) reset(headroom int) { *p = Packet{ - buffer: p.buffer, // keep the buffer - RawPacket: p.buffer[headroom:], // restore the full packet capacity (minus headroom). + buffer: p.buffer, // keep the buffer + RawPacket: p.buffer[headroom:], // restore the full packet capacity (minus headroom). + PriorityLabel: pr.WithBestEffort, // Default to best-effort. } // Everything else is reset to zero value. } From 3b49826322be7ad1ccea1e1c1a9c1f908c1e5dd1 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Fri, 30 Jan 2026 14:31:32 +0100 Subject: [PATCH 09/16] Fix: always increment packet drop counter when Send fails. --- router/dataplane.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/router/dataplane.go b/router/dataplane.go index 8a7fd48945..a1b354b2d1 100644 --- a/router/dataplane.go +++ b/router/dataplane.go @@ -895,6 +895,8 @@ func (d *dataPlane) runSlowPathProcessor(id int, q <-chan *Packet) { continue } if !egressLink.Send(p) { + sc := ClassOfSize(len(p.RawPacket)) + p.Link.Metrics()[sc].DroppedPacketsBusyForwarder.Inc() d.packetPool.Put(p) } } @@ -2222,6 +2224,8 @@ func (b *bfdSend) Send(bfd *layers.BFD) error { p.PriorityLabel = pr.WithPriority if !fwLink.Send(p) { + sc := ClassOfSize(len(p.RawPacket)) + fwLink.Metrics()[sc].DroppedPacketsBusyForwarder.Inc() // We do not care if some BFD packets get bounced under high load. If it becomes a problem, // the solution is do use BFD's demand-mode. To be considered in a future refactoring. b.dataPlane.packetPool.Put(p) From b5903d70ae3287700b417ee1247cb19c15d734b7 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Tue, 27 Jan 2026 12:18:15 +0100 Subject: [PATCH 10/16] WIP acceptance test --- demo/router_priority/BUILD.bazel | 12 + demo/router_priority/README.md | 81 +++++ demo/router_priority/test.py | 513 +++++++++++++++++++++++++++++++ 3 files changed, 606 insertions(+) create mode 100644 demo/router_priority/BUILD.bazel create mode 100644 demo/router_priority/README.md create mode 100755 demo/router_priority/test.py diff --git a/demo/router_priority/BUILD.bazel b/demo/router_priority/BUILD.bazel new file mode 100644 index 0000000000..9ceead5358 --- /dev/null +++ b/demo/router_priority/BUILD.bazel @@ -0,0 +1,12 @@ +load("//:scion.bzl", "scion_go_binary") +load("//acceptance/common:topogen.bzl", "topogen_test") + +topogen_test( + name = "test", + src = "test.py", + args = [ + ], + data = [ + ], + topo = "//topology:tiny.topo", +) diff --git a/demo/router_priority/README.md b/demo/router_priority/README.md new file mode 100644 index 0000000000..70f80fc9a8 --- /dev/null +++ b/demo/router_priority/README.md @@ -0,0 +1,81 @@ +# Border Router Forwarding Priority Test + +The test ensures that the priorities at the border router work as expected: +this means, that the packets flagged as priority are forwarded without losses, +with some caveats: +- All packets incoming to an AS can be read and processed by the border router. +- Any possible packet drops are due to lack of bandwidth on the egress interface + (this is a consequence of the previous point). +- Priority traffic does not exceed the capacity of the egress interface. + +Under these conditions, the packet prioritization done in the border router should +prevent any packet drops for priority traffic. + +The test checks for BFD packet drops, which are always flagged as priority, +in a controlled scenario: +- The border router has enough processing capacity: + - The test will limit the capacity of the network interfaces to a small bandwidth. + - The test uses the very small `Tiny.topo` topology, + which needs a small number of processes, which in turn do not consume much CPU. +- The priority traffic does not exceed the egress capacity: + - The amount of BFD traffic is configured in the test to be very small. + +This test uses the tiny topology: +```text + +-----------------+ + | | + | AS 1-ff00:0:110 | + | | + +-----------------+ + + + + + +-----------------+ +-----------------+ + | | | | + | AS 1-ff00:0:111 | | AS 1-ff00:0:112 | + | | | | + +-----------------+ +-----------------+ +``` + +## Components of the test +Out of the box from the tiny topology we have: +- 4 SCION border routers (the AS 1-ff00:0:110 has two BRs). +- 3 SCION control services. +- 3 SCION dispatchers for the control services. +- 3 SCION daemons. +- 3 tester applications. +- 3 SCION dispatchers for the tester applications. + +For a total of 19 docker containers. +Additionally, the tiny topology defines 5 networks. Here is the list and the containers using them: +- scn_000: Inter-AS 110 <-> 111 + - `br-1 @ 1-ff00:0:110` + - `br-1 @ 1-ff00:0:111` <--- This is the one we want to limit its capacity. +- scn_001: Intra-AS 110 + - `br-1 @ 1-ff00:0:110` + - `br-2 @ 1-ff00:0:110` + - `daemon @ 1-ff00:0:110` + - `disp-cs-1 @ 1-ff00:0:110` + - `disp-tester @ 1-ff00:0:110` +- scn_002: Intra-AS 111 + - `br-1 @ 1-ff00:0:111` + - `daemon @ 1-ff00:0:111` + - `disp-cs-1 @ 1-ff00:0:111` + - `disp-tester @ 1-ff00:0:111` +- scn_003: Inter-AS 110 <-> 112 + - `br-2 @ 1-ff00:0:110` + - `br-1 @ 1-ff00:0:112` +- scn_004: Intra-AS 112 + - `br-1 @ 1-ff00:0:112` + - `daemon @ 1-ff00:0:112` + - `disp-cs-1 @ 1-ff00:0:112` + - `disp-tester @ 1-ff00:0:112` + +The test introduces some changes to the docker compose file (modified via `test.py`), +so that `tc` is run to set bandwidth limits. + + +## How to run the test + +TODO \ No newline at end of file diff --git a/demo/router_priority/test.py b/demo/router_priority/test.py new file mode 100755 index 0000000000..781e5dade4 --- /dev/null +++ b/demo/router_priority/test.py @@ -0,0 +1,513 @@ +#!/usr/bin/env python3 + +# Copyright 2026 ETH Zurich +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from acceptance.common import base +from collections import defaultdict +from prometheus_client.parser import text_string_to_metric_families +from typing import Iterable, Dict, List, Tuple +import json +import re +import requests +import subprocess +import sys +import time +import yaml +# from plumbum import local + + +# deleteme remove once we run this as a test. +def selfdc(*args) -> str: + cmd = ["docker","compose", "-f", "gen/scion-dc.yml"] + list(args) + print(f"running {cmd}") + + try: + output = subprocess.run( + cmd, + text=True, + capture_output=True, + check=True, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"docker compose failed: {e.stderr}") from e + return output.stdout.strip() + + +def get_service_info(service: str) -> Dict: + output = selfdc("ps", "--format", "json") + # Currently, `docker compose top --format json` does NOT return a valid json string, but a + # bunch of lines containing valid json strings. Split into lines and treat them separately. + output = output.splitlines() + service_dict = None + for line in output: + ps_out = json.loads(line) + if ps_out["Service"] == service: + service_dict = ps_out + break + # If we found the service, get its PID and add it to the dictionary. + if service_dict is not None: + try: + pid = subprocess.check_output( + ["docker", "inspect", "-f", "{{.State.Pid}}", service_dict["Name"]], + text=True, + stderr=subprocess.PIPE, + ).strip() + except subprocess.CalledProcessError as e: + if "No such object" in e.stderr: + raise RuntimeError(f"Container '{service_dict["Name"]}' does not exist") from e + raise RuntimeError("Docker inspect failed") from e + service_dict["PID"] = pid + return service_dict + + +def get_interface_indices(service_name:str) -> List[int]: + """ + Returns the equivalent of running: + sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' container ) -n ip link + + :param container: Container name, e.g. scion-br1-ff00_0_111-1-1 + """ + # Step 1: get PID: info["PID"]. + service_info = get_service_info(service_name) + # Step 2: nsenter + ip link + proc = subprocess.run( + ["sudo", "nsenter", "-t", service_info["PID"], "-n", "ip", "link"], + text=True, + capture_output=True, + check=True + ) + # Step 3: parse output: the output contains 2 lines per interface, like this: + # 2: eth0@if751: mtu 1500 qdisc noqueue state UP mode DEFAULT group default + # link/ether 0e:40:6f:f3:6e:d7 brd ff:ff:ff:ff:ff:ff link-netnsid 0 + # We only need the first line, and only the second "field" of that line. + iface_re = re.compile(r'^\s*\d+:\s+([^:]+):') + interfaces = [ + m.group(1) + for line in proc.stdout.splitlines() + if (m := iface_re.match(line)) + ] + # Step 4: strip eth0@if out of the name of the interface: + index_re = re.compile(r'^eth\d+@if(\d+)') + indices = [ + int(m.group(1)) + for iface in interfaces + if (m:= index_re.match(iface)) + ] + return indices + + +def get_host_bridge_interface(indices: Iterable[int], network_name:str) -> str: + """Given the indices, it returns the first interface that matches the given network.""" + proc = subprocess.run( + ["ip", "-o", "link", "show"], + text=True, + capture_output=True, + check=True + ) + lines = proc.stdout.splitlines() + # Output is like: (lines intentionally broken with "\") + # 869: scn_000: mtu 1500 qdisc noqueue state UP mode \ + # DEFAULT group default \ link/ether 62:53:60:c8:e2:e5 brd ff:ff:ff:ff:ff:ff + # 870: scn_003: mtu 1500 qdisc noqueue state UP mode \ + # DEFAULT group default \ link/ether c2:b7:65:8b:c1:ef brd ff:ff:ff:ff:ff:ff + # 871: vethf9d48a7@if2: mtu 1500 qdisc noqueue \ + # master scn_003 state UP mode DEFAULT group default \ link/ether 86:e9:1d:77:c9:1f brd ff:ff:ff:ff:ff:ff link-netnsid 0 + # bridges_re = re.compile(r"^\s*(\d+):\s+([^\s:]+):.*?(?:\bmaster\s+(\S+))?") + # bridges_re = re.compile(r"^\s*(\d+):\s+([^\s:]+):.*master\s+(\S+)") + bridges_re = re.compile(r"^\s*(\d+):\s+([^\s@]+)@if\d+.*master\s+(\S+)") + for line in lines: + m = bridges_re.match(line) + if not m: + continue + idx = int(m.group(1)) + if not idx in indices: + continue + network = m.group(3) + if network == network_name: + return m.group(2) + raise RuntimeError(f"did not find the bridge for {network_name} and indices: {indices}") + + +def set_tc_limits(bridge:str, rate:str, burst:str, latency:str) -> None: + # try to reset the qdisc of the device. + try: + proc = subprocess.run( + ["sudo", "tc", "qdisc", "del", "dev", bridge, "root"], + check=True + ) + except subprocess.CalledProcessError as e: + # ignore failure (fails if not already set). + pass + + # set qdisc of the device to 1Mbps: + try: + subprocess.run( + ["sudo", "tc", "qdisc", "add", "dev", bridge, + "root", "tbf", "rate", rate, "burst", burst, "latency", latency], + text=True, + capture_output=True, + check=True + ) + except subprocess.CalledProcessError as e: + print(e.stderr) + raise RuntimeError(f"command tc failed: {e.stderr}") from e + + +def measure_br(url: str): + metrics = { + "router_bfd_state_changes":{ + "total": 0, + }, + "router_bfd_sent_packets":{ + "total": 0, + "interface": defaultdict(int), + }, + "router_bfd_received_packets":{ + "total": 0, + "interface": defaultdict(int), + }, + "router_dropped_pkts":{ + "total": 0, + "interface": defaultdict(int), + "reason": defaultdict(int), + }, + "router_output_pkts":{ + "total": 0, + "interface": defaultdict(int), + }, + } + text = requests.get(url).text + for family in text_string_to_metric_families(text): + if not family.name in metrics: + continue + metric = metrics[family.name] + for sample in family.samples: + # each sample has .value and .labels + metric["total"] += sample.value + # sample.labels is a dictionary like {'interface': '41', 'isd_as': '1-ff00:0:111'} + for label, label_value in sample.labels.items(): + if label in metric: + metric[label][label_value] += sample.value + return metrics + + + + + +def aggregate_metrics(full_text:str, metric_label:str) -> Tuple[int, Dict[str, int]]: + """returns the total and per interface aggregated (sum) metrics""" + total = 0 + by_interface = defaultdict(int) + + for family in text_string_to_metric_families(full_text): + if family.name != metric_label: + continue + + for sample in family.samples: + labels = sample.labels + value = sample.value + + total += value + by_interface[labels["interface"]] += value + + return total, dict(by_interface) + + +def measure_metrics_rate(url: str, sample_seconds:float, labels: List[str]) -> List[ + Tuple[float, Dict[str, float]]]: + # Get initial values: + text = requests.get(url).text + t0 = time.monotonic() + totals = [] # one per label + by_ifs = [] # one per label + for label in labels: + last_total, last_by_if = aggregate_metrics(text, label) + totals.append(last_total) + by_ifs.append(last_by_if) + + # Sleep. + time.sleep(sample_seconds) + + # Get new values: + text = requests.get(url).text + duration = time.monotonic() - t0 + for i in range(len(labels)): + label = labels[i] + total, by_if = aggregate_metrics(text, label) + # Relative to last measurement: + rel_total = total - totals[i] + rel_by_if = {k:by_if[k] - v for k,v in by_ifs[i].items()} + + # Compute average. + avg_total = rel_total / duration + avg_by_if = {k:v / duration for k, v in rel_by_if.items()} + + # Update lists: + totals[i] = avg_total + by_ifs[i] = avg_by_if + return totals, by_ifs + + +# def measure_metrics(): +# text = requests.get("http://172.20.0.26:30442/metrics").text +# labels = [ +# "router_bfd_sent_packets", +# "router_bfd_state_changes", +# "router_dropped_pkts", +# "router_output_pkts", +# "router_output_bytes", +# ] +# for label in labels: +# total, by_if = aggregate_metrics(text, label) +# print(f"{label} TOTAL :", total) +# print(f"{label} BY IF:", by_if) + +# totals, by_ifs = measure_metrics_rate( +# "http://172.20.0.26:30442/metrics", +# 2.0, +# labels, +# ) +# print("------- RATES ---------") +# for i in range(len(labels)): +# print(f"{labels[i]} total = {totals[i]}, by_if = {by_ifs[i]}") + + + + + + + + + + +def run_scion_ping(src_container:str, dst_endpoint:str, count:int, size:int, interval:str) -> float: + """Returns the loss rate 0..100""" + cmd = ["scion","ping","--format", "yaml", + "-c", str(count), "-s", str(size), "--interval", str(interval), dst_endpoint] + lines = selfdc("exec", src_container, *cmd) + ping = yaml.safe_load(lines) + stats = ping["statistics"] + print(f"sent: {stats["sent"]}") + print(f"recv: {stats["received"]}") + print(f"loss: {stats["packet_loss"]}") + return float(stats["packet_loss"]) + + +def run_heavy_scion_ping(src_container: str, dst_endpoint:str) -> float: + # docker compose -f gen/scion-dc.yml exec tester_1-ff00_0_111 scion ping --format yaml -c 3000 -s 2000 --interval 1ms 1-ff00:0:112,fd00:f00d:cafe::7f00:15 + loss = run_scion_ping(src_container,dst_endpoint,count=3000,size=2000, interval="1ms") + return loss + + +def increase_load(src_service:str, dst_endpoint: str, duration: str) -> None: + # go build -o sender ./demo/router_priority/sender/ && sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-tester_1-ff00_0_111-1 ) -n ./sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 -duration 60s + # deleteme run directly in the container after copying the binary with dc cp + service_info = get_service_info(src_service) + cmd = ["sudo", "nsenter", "-t", service_info["PID"], "-n", + "./sender", "-daemon", "172.20.0.28:30255", "-local", "172.20.0.29:0", + "-remote", dst_endpoint, + "-duration", duration ] + try: + subprocess.run( + cmd, + text=True, + capture_output=True, + check=True, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"docker compose failed: {e.stderr}") from e + + + +class Test(base.TestTopogen): + def setup_prepare(self): + super().setup_prepare() + + # We need to set up bandwidth limits to the interface of BR-1 @ 1-ff00:0:111, + # to ensure that we will see packet drops when running a high bandwidth test with + # scion ping. We should see no losses on priority traffic, e.g. BFD packets. + + # From the host, we obtain all the interfaces of the BR-1 @ 111 container: + + # sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-br1-ff00_0_111-1-1 ) -n ip link + # ip link | grep veth + + + # with (open(self.artifacts / "gen/scion-dc.yml", "r") as file): + # scion_dc = yaml.safe_load(file) + + # with open(self.artifacts / "gen/scion-dc.yml", "w") as file: + # yaml.dump(scion_dc, file) + + def _run(self): + print("deleteme running priority test") + print(f"interfaces: {get_interface_indices("scion-br1-ff00_0_111-1-1")}") + + +def deleteme(): + print("deleteme running priority test") + # sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-br1-ff00_0_111-1-1 ) -n ip link + # List interfaces of the BR-111 container. + indices = get_interface_indices("br1-ff00_0_111-1") + # ip -o link show + # Get the network interface for BR-111 -> BR-110 (network is scn_000). + bridge = get_host_bridge_interface(indices, "scn_000") + print(f"bridge is: {bridge}") + # sudo tc qdisc add dev veth31e7685@if3 root tbf rate 1mbit burst 32kbit latency 400ms + # Limit the BR-111 -> BR-110 interface to 1Mbps. + set_tc_limits(bridge, rate="1mbit", burst="32kbit", latency="400ms") + + # # curl http://172.20.0.26:30442/metrics | grep bfd_sent + # measure_metrics() + # # # Test: measure rates continuously: + # # labels = [ + # # "router_bfd_sent_packets", + # # "router_dropped_pkts", + # # "router_output_pkts", + # # "router_output_bytes", + # # ] + # # while True: + # # t0 = time.monotonic() + # # totals, by_ifs = measure_metrics_rate( + # # "http://172.20.0.26:30442/metrics", + # # 1.0, + # # labels, + # # ) + # # t1 = time.monotonic() + # # print(f"------- RATES --------- (after {(t1-t0):.3f} seconds)") + # # for i in range(len(labels)): + # # print(f"{labels[i]} total = {totals[i]}, by_if = {by_ifs[i]}") + + + # Increment bandwidth load significantly: + + # # --------------------- + # loss = run_heavy_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15") + # print(f"heavy traffic scion ping has a loss of {loss}") + # # Wait a bit and check that all works again + # time.sleep(1) + # loss = run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", + # count=3,size=1000,interval="1s") + # print(f"regular scion ping has a loss of {loss}") + # # -------------------------- + # + # go run ./demo/router_priority/sender/ -daemon 127.0.0.19:30255 -remote 1-ff00:0:110,127.0.0.1:12345 + # go run ./demo/router_priority/sender/ -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 + + # Build sender binary: + # go build -o sender ./demo/router_priority/sender/ + # Run sender but in the tester-111 network namespace: + # sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-tester_1-ff00_0_111-1 ) -n ./sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 + + # go build -o sender ./demo/router_priority/sender/ && sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-tester_1-ff00_0_111-1 ) -n ./sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 -duration 60s + + # Measure ping loss before loading the BR: + loss = run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", + count=3,size=1000,interval="1s") + print(f"initial ping loss is {loss}") + if loss > 90.0: + raise RuntimeError(f"The initial ping command has too high a loss ratio: {loss}") + # Measure BR-111 before increasing the load: + metrics_before = measure_br("http://172.20.0.26:30442/metrics") + + # Increase the load for 1 minute by blasting the destination with SCION UDP packets: + increase_load("tester_1-ff00_0_111", "1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345", "60s") + + # Ping again. + loss = run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", + count=3,size=1000,interval="1s") + print(f"final ping loss is {loss}") + if loss > 90.0: + print(f"The initial ping command has too high a loss ratio: {loss}") + sys.exit(1) + + # Measure BR-111 after the load increase: + metrics_after = measure_br("http://172.20.0.26:30442/metrics") + bfd_changes = metrics_after["router_bfd_state_changes"]["total"] -\ + metrics_before["router_bfd_state_changes"]["total"] + print(f"BFD state changes: {bfd_changes}") + if bfd_changes != 0: + print(f"BFD state should have not changed, but had {bfd_changes} changes.") + sys.exit(1) + busy_fwd = metrics_after["router_dropped_pkts"]["reason"]["busy_forwarder"] -\ + metrics_before["router_dropped_pkts"]["reason"]["busy_forwarder"] + if busy_fwd == 0: + print(f"Insufficient load: no packet drop occurred.") + sys.exit(1) + print(f"router metrics follow. Before:\n{metrics_before}\n" + f"After: {metrics_after}") + return + # scion ping -s 4000 --interval 1ms 1-ff00:0:112,fd00:f00d:cafe::7f00:15 + # scion ping -s 4000 1-ff00:0:112,fd00:f00d:cafe::7f00:15 + + # # This works: + # scion ping -s 1000 --interval 100ms 1-ff00:0:110,172.20.0.22 + + + + + + + +if __name__ == "__main__": + # base.main(Test) + deleteme() + # sudo tc qdisc add dev vethXYZ root tbf rate 1mbit burst 32kbit latency 400ms + + +# We need the following pip extra packages: +# - prometheus-client +# - requests + + +# docker compose -f gen/scion-dc.yml restart br1-ff00_0_111-1 +# OR +# scion.sh stop ; make && make docker-images && ./scion.sh start && sleep 10 && ./bin/end2end_integration -d + +# ip -o link show | grep scn_000 # BR111->BR110 +# sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-br1-ff00_0_111-1-1 ) -n ip address +# sudo tc qdisc add dev vethfb8dc0d root tbf rate 1mbit burst 32kbit latency 400ms + + +# docker compose -f gen/scion-dc.yml exec -it tester_1-ff00_0_111 bash + +# sudo tcpdump -i any 'udp' -w sender.pcap ; sudo chown juan:juan sender.pcap + + +# docker compose -f gen/scion-dc.yml logs br1-ff00_0_111-1 -f +# curl -s http://172.20.0.26:30442/metrics | grep bfd_sent +# curl -s http://172.20.0.26:30442/metrics | grep bfd +# curl -s http://172.20.0.26:30442/metrics | grep drop +# curl -s http://172.20.0.26:30442/metrics | grep processed_pkts + +# Blast the router with SCION UDP: +# go build -o sender ./demo/router_priority/sender/ && sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-tester_1-ff00_0_111-1 ) -n ./sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 -duration 60s + + +# Check that still works after the blast: +# docker compose -f gen/scion-dc.yml exec tester_1-ff00_0_111 scion ping -c 3 1-ff00:0:112,fd00:f00d:cafe::7f00:15 + + +""" +172.20.0.26 BR-1 @ 111 +172.20.0.27 disp CS @ 111 +172.20.0.28 daemon @ 111 +172.20.0.29 tester @ 111 + +ip.addr == 172.20.0.0/16 and ip.src==172.20.0.3 +udp && scion && scion.next_hdr == 202 && scmp.type +udp && scion && scion.src_host == "172.20.0.29" && scion.payload_len == 1108 && scion_udp.dst_port == 12345 + +""" From 1ceb8a8266963e66759da5a13278326a62bbbee2 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Fri, 30 Jan 2026 14:55:45 +0100 Subject: [PATCH 11/16] WIP Adding UDP blaster to the demo test. --- demo/router_priority/BUILD.bazel | 1 + demo/router_priority/sender/BUILD.bazel | 15 ++ demo/router_priority/sender/sender.go | 114 +++++++++++++++ demo/router_priority/test.py | 182 ++---------------------- 4 files changed, 142 insertions(+), 170 deletions(-) create mode 100644 demo/router_priority/sender/BUILD.bazel create mode 100644 demo/router_priority/sender/sender.go diff --git a/demo/router_priority/BUILD.bazel b/demo/router_priority/BUILD.bazel index 9ceead5358..3355233b05 100644 --- a/demo/router_priority/BUILD.bazel +++ b/demo/router_priority/BUILD.bazel @@ -7,6 +7,7 @@ topogen_test( args = [ ], data = [ + "//demo/router_priority/sender", ], topo = "//topology:tiny.topo", ) diff --git a/demo/router_priority/sender/BUILD.bazel b/demo/router_priority/sender/BUILD.bazel new file mode 100644 index 0000000000..eabd8ec860 --- /dev/null +++ b/demo/router_priority/sender/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["sender.go"], + importpath = "github.com/scionproto/scion/demo/router_priority/sender", + visibility = ["//visibility:private"], + deps = ["//pkg/snet:go_default_library"], +) + +go_binary( + name = "sender", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/demo/router_priority/sender/sender.go b/demo/router_priority/sender/sender.go new file mode 100644 index 0000000000..2d772d264c --- /dev/null +++ b/demo/router_priority/sender/sender.go @@ -0,0 +1,114 @@ +// Copyright 2026 ETH Zurich +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "fmt" + "net" + "time" + + "github.com/scionproto/scion/pkg/daemon" + "github.com/scionproto/scion/pkg/snet" + "github.com/scionproto/scion/pkg/snet/metrics" +) + +// -daemon 127.0.0.19:30255 -remote 1-ff00:0:110,172.20.0.18:12345 +func main() { + ctx, cancelF := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelF() + + var daemonAddr, localAddr, blastDuration string + var remote snet.UDPAddr + var payloadSize int + + flag.StringVar(&daemonAddr, "daemon", "", "local daemon address") + flag.StringVar(&localAddr, "local", "127.0.0.1:0", "local address") + flag.Var(&remote, "remote", "address to send to") + flag.StringVar(&blastDuration, "duration", "1s", "duration of the SCION UDP blast") + flag.IntVar(&payloadSize, "payloadsize", 1100, "size of the payload in bytes") + flag.Parse() + + // Duration: + duration, err := time.ParseDuration(blastDuration) + panicOnError(err) + + // Find daemon. + daemonConn, err := daemon.NewService(daemonAddr).Connect(ctx) + panicOnError(err) + + fmt.Printf("remote: %s\n", &remote) + + // Where am I? + localIA, err := daemonConn.LocalIA(ctx) + panicOnError(err) + // local := net.UDPAddrFromAddrPort( + // netip.AddrPortFrom(netip.AddrFrom4([4]byte{127, 0, 0, 1}), + // 0)) + local, err := net.ResolveUDPAddr("udp", localAddr) + panicOnError(err) + fmt.Printf("On local IA: %s, host: %s\n", localIA, local) + + // Network: + metrics := metrics.NewSCIONPacketConnMetrics() + topo, err := daemon.LoadTopology(ctx, daemonConn) + panicOnError(err) + network := &snet.SCIONNetwork{ + SCMPHandler: snet.DefaultSCMPHandler{ + RevocationHandler: daemon.RevHandler{Connector: daemonConn}, + SCMPErrors: metrics.SCMPErrors, + }, + PacketConnMetrics: metrics, + Topology: topo, + } + + // Get paths. + paths, err := daemonConn.Paths(ctx, remote.IA, localIA, daemon.PathReqFlags{}) + panicOnError(err) + if len(paths) == 0 { + panic("no paths") + } + path := paths[0] + remote.Path = path.Dataplane() + remote.NextHop = path.UnderlayNextHop() + + // Blast the remote endpoint with packets. + conn, err := network.Dial(ctx, "udp", local, &remote) + panicOnError(err) + payload := make([]byte, payloadSize) + var packetCount int + t0 := time.Now() + tLastLog := time.Now() + for packetCount = 0; ; packetCount++ { + n, err := conn.Write(payload) + panicOnError(err) + if n != len(payload) { + panic(fmt.Errorf("only sent %d out of %d", n, len(payload))) + } + if time.Since(tLastLog) >= time.Second { + fmt.Printf("sent %d packets (%d bytes payload) to %s\n", + packetCount, len(payload), remote.String()) + tLastLog = time.Now() + } + if time.Since(t0) >= duration { + break + } + } + fmt.Printf("sent %d packets in total.\n", packetCount) +} + +func panicOnError(err error) { + if err != nil { + panic(err) + } +} diff --git a/demo/router_priority/test.py b/demo/router_priority/test.py index 781e5dade4..a5be703f15 100755 --- a/demo/router_priority/test.py +++ b/demo/router_priority/test.py @@ -25,7 +25,7 @@ import sys import time import yaml -# from plumbum import local +from plumbum import local # deleteme remove once we run this as a test. @@ -203,95 +203,6 @@ def measure_br(url: str): return metrics - - - -def aggregate_metrics(full_text:str, metric_label:str) -> Tuple[int, Dict[str, int]]: - """returns the total and per interface aggregated (sum) metrics""" - total = 0 - by_interface = defaultdict(int) - - for family in text_string_to_metric_families(full_text): - if family.name != metric_label: - continue - - for sample in family.samples: - labels = sample.labels - value = sample.value - - total += value - by_interface[labels["interface"]] += value - - return total, dict(by_interface) - - -def measure_metrics_rate(url: str, sample_seconds:float, labels: List[str]) -> List[ - Tuple[float, Dict[str, float]]]: - # Get initial values: - text = requests.get(url).text - t0 = time.monotonic() - totals = [] # one per label - by_ifs = [] # one per label - for label in labels: - last_total, last_by_if = aggregate_metrics(text, label) - totals.append(last_total) - by_ifs.append(last_by_if) - - # Sleep. - time.sleep(sample_seconds) - - # Get new values: - text = requests.get(url).text - duration = time.monotonic() - t0 - for i in range(len(labels)): - label = labels[i] - total, by_if = aggregate_metrics(text, label) - # Relative to last measurement: - rel_total = total - totals[i] - rel_by_if = {k:by_if[k] - v for k,v in by_ifs[i].items()} - - # Compute average. - avg_total = rel_total / duration - avg_by_if = {k:v / duration for k, v in rel_by_if.items()} - - # Update lists: - totals[i] = avg_total - by_ifs[i] = avg_by_if - return totals, by_ifs - - -# def measure_metrics(): -# text = requests.get("http://172.20.0.26:30442/metrics").text -# labels = [ -# "router_bfd_sent_packets", -# "router_bfd_state_changes", -# "router_dropped_pkts", -# "router_output_pkts", -# "router_output_bytes", -# ] -# for label in labels: -# total, by_if = aggregate_metrics(text, label) -# print(f"{label} TOTAL :", total) -# print(f"{label} BY IF:", by_if) - -# totals, by_ifs = measure_metrics_rate( -# "http://172.20.0.26:30442/metrics", -# 2.0, -# labels, -# ) -# print("------- RATES ---------") -# for i in range(len(labels)): -# print(f"{labels[i]} total = {totals[i]}, by_if = {by_ifs[i]}") - - - - - - - - - - def run_scion_ping(src_container:str, dst_endpoint:str, count:int, size:int, interval:str) -> float: """Returns the loss rate 0..100""" cmd = ["scion","ping","--format", "yaml", @@ -299,18 +210,9 @@ def run_scion_ping(src_container:str, dst_endpoint:str, count:int, size:int, int lines = selfdc("exec", src_container, *cmd) ping = yaml.safe_load(lines) stats = ping["statistics"] - print(f"sent: {stats["sent"]}") - print(f"recv: {stats["received"]}") - print(f"loss: {stats["packet_loss"]}") return float(stats["packet_loss"]) -def run_heavy_scion_ping(src_container: str, dst_endpoint:str) -> float: - # docker compose -f gen/scion-dc.yml exec tester_1-ff00_0_111 scion ping --format yaml -c 3000 -s 2000 --interval 1ms 1-ff00:0:112,fd00:f00d:cafe::7f00:15 - loss = run_scion_ping(src_container,dst_endpoint,count=3000,size=2000, interval="1ms") - return loss - - def increase_load(src_service:str, dst_endpoint: str, duration: str) -> None: # go build -o sender ./demo/router_priority/sender/ && sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-tester_1-ff00_0_111-1 ) -n ./sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 -duration 60s # deleteme run directly in the container after copying the binary with dc cp @@ -335,25 +237,13 @@ class Test(base.TestTopogen): def setup_prepare(self): super().setup_prepare() - # We need to set up bandwidth limits to the interface of BR-1 @ 1-ff00:0:111, - # to ensure that we will see packet drops when running a high bandwidth test with - # scion ping. We should see no losses on priority traffic, e.g. BFD packets. - - # From the host, we obtain all the interfaces of the BR-1 @ 111 container: - - # sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-br1-ff00_0_111-1-1 ) -n ip link - # ip link | grep veth - - - # with (open(self.artifacts / "gen/scion-dc.yml", "r") as file): - # scion_dc = yaml.safe_load(file) - - # with open(self.artifacts / "gen/scion-dc.yml", "w") as file: - # yaml.dump(scion_dc, file) - def _run(self): - print("deleteme running priority test") - print(f"interfaces: {get_interface_indices("scion-br1-ff00_0_111-1-1")}") + print("deleteme running router_priority test") + self.await_connectivity() + + sender_bin = local["realpath"](self.get_executable("sender").executable).strip() + self.dc("cp", sender_bin, "tester_1-ff00_0_111" + ":/bin/") + print(f"deleteme finished router_priority test") def deleteme(): @@ -367,51 +257,7 @@ def deleteme(): print(f"bridge is: {bridge}") # sudo tc qdisc add dev veth31e7685@if3 root tbf rate 1mbit burst 32kbit latency 400ms # Limit the BR-111 -> BR-110 interface to 1Mbps. - set_tc_limits(bridge, rate="1mbit", burst="32kbit", latency="400ms") - - # # curl http://172.20.0.26:30442/metrics | grep bfd_sent - # measure_metrics() - # # # Test: measure rates continuously: - # # labels = [ - # # "router_bfd_sent_packets", - # # "router_dropped_pkts", - # # "router_output_pkts", - # # "router_output_bytes", - # # ] - # # while True: - # # t0 = time.monotonic() - # # totals, by_ifs = measure_metrics_rate( - # # "http://172.20.0.26:30442/metrics", - # # 1.0, - # # labels, - # # ) - # # t1 = time.monotonic() - # # print(f"------- RATES --------- (after {(t1-t0):.3f} seconds)") - # # for i in range(len(labels)): - # # print(f"{labels[i]} total = {totals[i]}, by_if = {by_ifs[i]}") - - - # Increment bandwidth load significantly: - - # # --------------------- - # loss = run_heavy_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15") - # print(f"heavy traffic scion ping has a loss of {loss}") - # # Wait a bit and check that all works again - # time.sleep(1) - # loss = run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", - # count=3,size=1000,interval="1s") - # print(f"regular scion ping has a loss of {loss}") - # # -------------------------- - # - # go run ./demo/router_priority/sender/ -daemon 127.0.0.19:30255 -remote 1-ff00:0:110,127.0.0.1:12345 - # go run ./demo/router_priority/sender/ -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 - - # Build sender binary: - # go build -o sender ./demo/router_priority/sender/ - # Run sender but in the tester-111 network namespace: - # sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-tester_1-ff00_0_111-1 ) -n ./sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 - - # go build -o sender ./demo/router_priority/sender/ && sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-tester_1-ff00_0_111-1 ) -n ./sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 -duration 60s + set_tc_limits(bridge, rate="512kbit", burst="32kbit", latency="400ms") # Measure ping loss before loading the BR: loss = run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", @@ -446,14 +292,10 @@ def deleteme(): if busy_fwd == 0: print(f"Insufficient load: no packet drop occurred.") sys.exit(1) - print(f"router metrics follow. Before:\n{metrics_before}\n" - f"After: {metrics_after}") - return - # scion ping -s 4000 --interval 1ms 1-ff00:0:112,fd00:f00d:cafe::7f00:15 - # scion ping -s 4000 1-ff00:0:112,fd00:f00d:cafe::7f00:15 - - # # This works: - # scion ping -s 1000 --interval 100ms 1-ff00:0:110,172.20.0.22 + print(f"router metrics follow.\n" + f"Before: -----8<-----\n{metrics_before}\n-----8<-----\n" + f"After: -----8<-----\n{metrics_after}\n-----8<-----") + print("Success.") From 90164f89c0b3ee36e02f0a8fa600d85c97f18ce9 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Mon, 2 Feb 2026 15:03:05 +0100 Subject: [PATCH 12/16] Finish router priority demo test. --- MODULE.bazel.lock | 70 +++- demo/router_priority/BUILD.bazel | 6 + demo/router_priority/README.md | 25 +- demo/router_priority/sender/BUILD.bazel | 8 +- .../sender/{sender.go => main.go} | 9 +- demo/router_priority/test.py | 304 ++++++------------ tools/env/pip3/requirements.in | 2 + tools/env/pip3/requirements.txt | 135 ++++++++ 8 files changed, 341 insertions(+), 218 deletions(-) rename demo/router_priority/sender/{sender.go => main.go} (92%) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 0776871b69..58f20cac0e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -3722,6 +3722,33 @@ ] } }, + "scion_python_deps_312_certifi": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@scion_python_deps//{name}:{target}", + "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", + "repo": "scion_python_deps_312", + "requirement": "certifi==2026.1.4 --hash=sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c --hash=sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120" + } + }, + "scion_python_deps_312_charset_normalizer": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@scion_python_deps//{name}:{target}", + "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", + "repo": "scion_python_deps_312", + "requirement": "charset-normalizer==3.4.4 --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608" + } + }, + "scion_python_deps_312_idna": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@scion_python_deps//{name}:{target}", + "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", + "repo": "scion_python_deps_312", + "requirement": "idna==3.11 --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902" + } + }, "scion_python_deps_312_plumbum": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -3731,6 +3758,15 @@ "requirement": "plumbum==1.6.9 --hash=sha256:16b9e19d96c80f2e9d051ef5f04927b834a6ac0ce5d2768eb8662b5cd53e43df --hash=sha256:91418dcc66b58ab9d2e3b04b3d1e0d787dc45923154fb8b4a826bd9316dba0d6" } }, + "scion_python_deps_312_prometheus_client": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@scion_python_deps//{name}:{target}", + "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", + "repo": "scion_python_deps_312", + "requirement": "prometheus-client==0.24.1 --hash=sha256:150db128af71a5c2482b36e588fc8a6b95e498750da4b17065947c16070f4055 --hash=sha256:7e0ced7fbbd40f7b84962d5d2ab6f17ef88a72504dcf7c0b40737b43b2a461f9" + } + }, "scion_python_deps_312_pyyaml": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -3740,6 +3776,15 @@ "requirement": "pyyaml==6.0.1 --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" } }, + "scion_python_deps_312_requests": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@scion_python_deps//{name}:{target}", + "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", + "repo": "scion_python_deps_312", + "requirement": "requests==2.32.5 --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf" + } + }, "scion_python_deps_312_setuptools": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -3785,6 +3830,15 @@ "requirement": "toml==0.10.2 --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" } }, + "scion_python_deps_312_urllib3": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@scion_python_deps//{name}:{target}", + "python_interpreter_target": "@@rules_python++python+python_3_12_host//:python", + "repo": "scion_python_deps_312", + "requirement": "urllib3==2.6.3 --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4" + } + }, "scion_python_doc_deps_312_alabaster": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -4404,22 +4458,34 @@ "repo_name": "scion_python_deps", "extra_hub_aliases": {}, "whl_map": { + "certifi": "{\"scion_python_deps_312_certifi\":[{\"version\":\"3.12\"}]}", + "charset_normalizer": "{\"scion_python_deps_312_charset_normalizer\":[{\"version\":\"3.12\"}]}", + "idna": "{\"scion_python_deps_312_idna\":[{\"version\":\"3.12\"}]}", "plumbum": "{\"scion_python_deps_312_plumbum\":[{\"version\":\"3.12\"}]}", + "prometheus_client": "{\"scion_python_deps_312_prometheus_client\":[{\"version\":\"3.12\"}]}", "pyyaml": "{\"scion_python_deps_312_pyyaml\":[{\"version\":\"3.12\"}]}", + "requests": "{\"scion_python_deps_312_requests\":[{\"version\":\"3.12\"}]}", "setuptools": "{\"scion_python_deps_312_setuptools\":[{\"version\":\"3.12\"}]}", "six": "{\"scion_python_deps_312_six\":[{\"version\":\"3.12\"}]}", "supervisor": "{\"scion_python_deps_312_supervisor\":[{\"version\":\"3.12\"}]}", "supervisor_wildcards": "{\"scion_python_deps_312_supervisor_wildcards\":[{\"version\":\"3.12\"}]}", - "toml": "{\"scion_python_deps_312_toml\":[{\"version\":\"3.12\"}]}" + "toml": "{\"scion_python_deps_312_toml\":[{\"version\":\"3.12\"}]}", + "urllib3": "{\"scion_python_deps_312_urllib3\":[{\"version\":\"3.12\"}]}" }, "packages": [ + "certifi", + "charset_normalizer", + "idna", "plumbum", + "prometheus_client", "pyyaml", + "requests", "setuptools", "six", "supervisor", "supervisor_wildcards", - "toml" + "toml", + "urllib3" ], "groups": {} } diff --git a/demo/router_priority/BUILD.bazel b/demo/router_priority/BUILD.bazel index 3355233b05..0f1c7304f7 100644 --- a/demo/router_priority/BUILD.bazel +++ b/demo/router_priority/BUILD.bazel @@ -1,10 +1,16 @@ +load("@scion_python_deps//:requirements.bzl", "requirement") load("//:scion.bzl", "scion_go_binary") load("//acceptance/common:topogen.bzl", "topogen_test") topogen_test( name = "test", src = "test.py", + deps = [ + requirement("prometheus-client"), + requirement("requests"), + ], args = [ + "--executable=sender:$(location //demo/router_priority/sender)", ], data = [ "//demo/router_priority/sender", diff --git a/demo/router_priority/README.md b/demo/router_priority/README.md index 70f80fc9a8..7f4dcb82f1 100644 --- a/demo/router_priority/README.md +++ b/demo/router_priority/README.md @@ -16,7 +16,7 @@ in a controlled scenario: - The border router has enough processing capacity: - The test will limit the capacity of the network interfaces to a small bandwidth. - The test uses the very small `Tiny.topo` topology, - which needs a small number of processes, which in turn do not consume much CPU. + which needs a very small number of processes, which in turn do not consume much CPU. - The priority traffic does not exceed the egress capacity: - The amount of BFD traffic is configured in the test to be very small. @@ -27,14 +27,16 @@ This test uses the tiny topology: | AS 1-ff00:0:110 | | | +-----------------+ - - - - + | + +--------------------+--------------------+ + | | + | <---- capped interface | +-----------------+ +-----------------+ | | | | | AS 1-ff00:0:111 | | AS 1-ff00:0:112 | | | | | + | tester-111 | | tester-112 | + | | | | +-----------------+ +-----------------+ ``` @@ -51,7 +53,7 @@ For a total of 19 docker containers. Additionally, the tiny topology defines 5 networks. Here is the list and the containers using them: - scn_000: Inter-AS 110 <-> 111 - `br-1 @ 1-ff00:0:110` - - `br-1 @ 1-ff00:0:111` <--- This is the one we want to limit its capacity. + - `br-1 @ 1-ff00:0:111` <--- This is the one whose capacity we want to limit. - scn_001: Intra-AS 110 - `br-1 @ 1-ff00:0:110` - `br-2 @ 1-ff00:0:110` @@ -72,10 +74,13 @@ Additionally, the tiny topology defines 5 networks. Here is the list and the con - `disp-cs-1 @ 1-ff00:0:112` - `disp-tester @ 1-ff00:0:112` -The test introduces some changes to the docker compose file (modified via `test.py`), -so that `tc` is run to set bandwidth limits. +The test requires a sender binary (automatically built if run with bazel). +It is used at the tester-111 container to blast tester-112 with SCION UDP packets, +without any kind of flow control. +The regular `scion ping` does not work here, as it paces itself if responses are not received. -## How to run the test +## How to run the demo test -TODO \ No newline at end of file +1. [Set up the development environment](https://docs.scion.org/en/latest/build/setup.html) +2. `bazel test --test_output=streamed --cache_test_results=no //demo/router_priority:test` diff --git a/demo/router_priority/sender/BUILD.bazel b/demo/router_priority/sender/BUILD.bazel index eabd8ec860..7f3e9814d3 100644 --- a/demo/router_priority/sender/BUILD.bazel +++ b/demo/router_priority/sender/BUILD.bazel @@ -2,10 +2,14 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", - srcs = ["sender.go"], + srcs = ["main.go"], importpath = "github.com/scionproto/scion/demo/router_priority/sender", visibility = ["//visibility:private"], - deps = ["//pkg/snet:go_default_library"], + deps = [ + "//pkg/daemon:go_default_library", + "//pkg/snet:go_default_library", + "//pkg/snet/metrics:go_default_library", + ], ) go_binary( diff --git a/demo/router_priority/sender/sender.go b/demo/router_priority/sender/main.go similarity index 92% rename from demo/router_priority/sender/sender.go rename to demo/router_priority/sender/main.go index 2d772d264c..c7eaf2ab25 100644 --- a/demo/router_priority/sender/sender.go +++ b/demo/router_priority/sender/main.go @@ -88,23 +88,18 @@ func main() { payload := make([]byte, payloadSize) var packetCount int t0 := time.Now() - tLastLog := time.Now() for packetCount = 0; ; packetCount++ { n, err := conn.Write(payload) panicOnError(err) if n != len(payload) { panic(fmt.Errorf("only sent %d out of %d", n, len(payload))) } - if time.Since(tLastLog) >= time.Second { - fmt.Printf("sent %d packets (%d bytes payload) to %s\n", - packetCount, len(payload), remote.String()) - tLastLog = time.Now() - } if time.Since(t0) >= duration { break } } - fmt.Printf("sent %d packets in total.\n", packetCount) + fmt.Printf("sent %d packets (%d bytes payload) in total to %s.\n", + packetCount, len(payload), remote.String()) } func panicOnError(err error) { diff --git a/demo/router_priority/test.py b/demo/router_priority/test.py index a5be703f15..0f797efb24 100755 --- a/demo/router_priority/test.py +++ b/demo/router_priority/test.py @@ -28,68 +28,23 @@ from plumbum import local -# deleteme remove once we run this as a test. -def selfdc(*args) -> str: - cmd = ["docker","compose", "-f", "gen/scion-dc.yml"] + list(args) - print(f"running {cmd}") - - try: - output = subprocess.run( - cmd, - text=True, - capture_output=True, - check=True, - ) - except subprocess.CalledProcessError as e: - raise RuntimeError(f"docker compose failed: {e.stderr}") from e - return output.stdout.strip() - - -def get_service_info(service: str) -> Dict: - output = selfdc("ps", "--format", "json") - # Currently, `docker compose top --format json` does NOT return a valid json string, but a - # bunch of lines containing valid json strings. Split into lines and treat them separately. - output = output.splitlines() - service_dict = None - for line in output: - ps_out = json.loads(line) - if ps_out["Service"] == service: - service_dict = ps_out - break - # If we found the service, get its PID and add it to the dictionary. - if service_dict is not None: - try: - pid = subprocess.check_output( - ["docker", "inspect", "-f", "{{.State.Pid}}", service_dict["Name"]], - text=True, - stderr=subprocess.PIPE, - ).strip() - except subprocess.CalledProcessError as e: - if "No such object" in e.stderr: - raise RuntimeError(f"Container '{service_dict["Name"]}' does not exist") from e - raise RuntimeError("Docker inspect failed") from e - service_dict["PID"] = pid - return service_dict - - -def get_interface_indices(service_name:str) -> List[int]: +def get_interface_indices(service_info:Dict) -> List[int]: """ Returns the equivalent of running: sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' container ) -n ip link :param container: Container name, e.g. scion-br1-ff00_0_111-1-1 """ - # Step 1: get PID: info["PID"]. - service_info = get_service_info(service_name) - # Step 2: nsenter + ip link + # With nsenter, find the interfaces of the container. proc = subprocess.run( ["sudo", "nsenter", "-t", service_info["PID"], "-n", "ip", "link"], text=True, capture_output=True, check=True ) - # Step 3: parse output: the output contains 2 lines per interface, like this: - # 2: eth0@if751: mtu 1500 qdisc noqueue state UP mode DEFAULT group default + # Parse output: the output contains 2 lines per interface, like this: (first line broken at \) + # 2: eth0@if751: mtu 1500 \ + # qdisc noqueue state UP mode DEFAULT group default # link/ether 0e:40:6f:f3:6e:d7 brd ff:ff:ff:ff:ff:ff link-netnsid 0 # We only need the first line, and only the second "field" of that line. iface_re = re.compile(r'^\s*\d+:\s+([^:]+):') @@ -98,7 +53,7 @@ def get_interface_indices(service_name:str) -> List[int]: for line in proc.stdout.splitlines() if (m := iface_re.match(line)) ] - # Step 4: strip eth0@if out of the name of the interface: + # Strip eth0@if out of the name of the interface: index_re = re.compile(r'^eth\d+@if(\d+)') indices = [ int(m.group(1)) @@ -124,8 +79,6 @@ def get_host_bridge_interface(indices: Iterable[int], network_name:str) -> str: # DEFAULT group default \ link/ether c2:b7:65:8b:c1:ef brd ff:ff:ff:ff:ff:ff # 871: vethf9d48a7@if2: mtu 1500 qdisc noqueue \ # master scn_003 state UP mode DEFAULT group default \ link/ether 86:e9:1d:77:c9:1f brd ff:ff:ff:ff:ff:ff link-netnsid 0 - # bridges_re = re.compile(r"^\s*(\d+):\s+([^\s:]+):.*?(?:\bmaster\s+(\S+))?") - # bridges_re = re.compile(r"^\s*(\d+):\s+([^\s:]+):.*master\s+(\S+)") bridges_re = re.compile(r"^\s*(\d+):\s+([^\s@]+)@if\d+.*master\s+(\S+)") for line in lines: m = bridges_re.match(line) @@ -141,17 +94,18 @@ def get_host_bridge_interface(indices: Iterable[int], network_name:str) -> str: def set_tc_limits(bridge:str, rate:str, burst:str, latency:str) -> None: - # try to reset the qdisc of the device. + # Try to reset the qdisc of the device. try: - proc = subprocess.run( + subprocess.run( ["sudo", "tc", "qdisc", "del", "dev", bridge, "root"], - check=True + check=True, + capture_output=True, ) except subprocess.CalledProcessError as e: - # ignore failure (fails if not already set). + # Ignore failure (fails if not already set). pass - # set qdisc of the device to 1Mbps: + # Set qdisc of the device to 1Mbps: try: subprocess.run( ["sudo", "tc", "qdisc", "add", "dev", bridge, @@ -194,7 +148,7 @@ def measure_br(url: str): continue metric = metrics[family.name] for sample in family.samples: - # each sample has .value and .labels + # Each sample has .value and .labels metric["total"] += sample.value # sample.labels is a dictionary like {'interface': '41', 'isd_as': '1-ff00:0:111'} for label, label_value in sample.labels.items(): @@ -203,153 +157,109 @@ def measure_br(url: str): return metrics -def run_scion_ping(src_container:str, dst_endpoint:str, count:int, size:int, interval:str) -> float: - """Returns the loss rate 0..100""" - cmd = ["scion","ping","--format", "yaml", - "-c", str(count), "-s", str(size), "--interval", str(interval), dst_endpoint] - lines = selfdc("exec", src_container, *cmd) - ping = yaml.safe_load(lines) - stats = ping["statistics"] - return float(stats["packet_loss"]) - - -def increase_load(src_service:str, dst_endpoint: str, duration: str) -> None: - # go build -o sender ./demo/router_priority/sender/ && sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-tester_1-ff00_0_111-1 ) -n ./sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 -duration 60s - # deleteme run directly in the container after copying the binary with dc cp - service_info = get_service_info(src_service) - cmd = ["sudo", "nsenter", "-t", service_info["PID"], "-n", - "./sender", "-daemon", "172.20.0.28:30255", "-local", "172.20.0.29:0", - "-remote", dst_endpoint, - "-duration", duration ] - try: - subprocess.run( - cmd, - text=True, - capture_output=True, - check=True, - ) - except subprocess.CalledProcessError as e: - raise RuntimeError(f"docker compose failed: {e.stderr}") from e - - - class Test(base.TestTopogen): def setup_prepare(self): super().setup_prepare() def _run(self): - print("deleteme running router_priority test") + print("-------------------- running router_priority test") self.await_connectivity() - + # Copy the sender binary to the tester-111 container (used to apply load). sender_bin = local["realpath"](self.get_executable("sender").executable).strip() self.dc("cp", sender_bin, "tester_1-ff00_0_111" + ":/bin/") - print(f"deleteme finished router_priority test") - - -def deleteme(): - print("deleteme running priority test") - # sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-br1-ff00_0_111-1-1 ) -n ip link - # List interfaces of the BR-111 container. - indices = get_interface_indices("br1-ff00_0_111-1") - # ip -o link show - # Get the network interface for BR-111 -> BR-110 (network is scn_000). - bridge = get_host_bridge_interface(indices, "scn_000") - print(f"bridge is: {bridge}") - # sudo tc qdisc add dev veth31e7685@if3 root tbf rate 1mbit burst 32kbit latency 400ms - # Limit the BR-111 -> BR-110 interface to 1Mbps. - set_tc_limits(bridge, rate="512kbit", burst="32kbit", latency="400ms") - - # Measure ping loss before loading the BR: - loss = run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", - count=3,size=1000,interval="1s") - print(f"initial ping loss is {loss}") - if loss > 90.0: - raise RuntimeError(f"The initial ping command has too high a loss ratio: {loss}") - # Measure BR-111 before increasing the load: - metrics_before = measure_br("http://172.20.0.26:30442/metrics") - - # Increase the load for 1 minute by blasting the destination with SCION UDP packets: - increase_load("tester_1-ff00_0_111", "1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345", "60s") - - # Ping again. - loss = run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", - count=3,size=1000,interval="1s") - print(f"final ping loss is {loss}") - if loss > 90.0: - print(f"The initial ping command has too high a loss ratio: {loss}") - sys.exit(1) - - # Measure BR-111 after the load increase: - metrics_after = measure_br("http://172.20.0.26:30442/metrics") - bfd_changes = metrics_after["router_bfd_state_changes"]["total"] -\ - metrics_before["router_bfd_state_changes"]["total"] - print(f"BFD state changes: {bfd_changes}") - if bfd_changes != 0: - print(f"BFD state should have not changed, but had {bfd_changes} changes.") - sys.exit(1) - busy_fwd = metrics_after["router_dropped_pkts"]["reason"]["busy_forwarder"] -\ - metrics_before["router_dropped_pkts"]["reason"]["busy_forwarder"] - if busy_fwd == 0: - print(f"Insufficient load: no packet drop occurred.") - sys.exit(1) - print(f"router metrics follow.\n" - f"Before: -----8<-----\n{metrics_before}\n-----8<-----\n" - f"After: -----8<-----\n{metrics_after}\n-----8<-----") - print("Success.") - - - - + # Get the BR-111 service information. + service_info = self._get_service_info("br1-ff00_0_111-1") + # List interfaces of the BR-111 container. + indices = get_interface_indices(service_info) + # Match those interfaces with one of the host interfaces. + bridge = get_host_bridge_interface(indices, "scn_000") + print(f"bridge is: {bridge}") + # Limit the BR-111 -> BR-110 interface to 1Mbps. + set_tc_limits(bridge, rate="512kbit", burst="32kbit", latency="400ms") + print(f"tc limits applied to host interface {bridge} (scn_000: BR-110 <-> BR-111)") + + # Measure ping loss before loading the BR: + loss = self._run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", + count=3,size=1000,interval="1s") + print(f"initial ping loss is {loss}") + if loss > 90.0: + raise RuntimeError(f"The initial ping command has too high a loss ratio: {loss}") + # Measure BR-111 before increasing the load: + metrics_before = measure_br("http://172.20.0.26:30442/metrics") + + # Increase the load for 1 minute by blasting the destination with SCION UDP packets: + # result = self.dc.execute("tester_1-ff00_0_111", + result = self.dc("exec", "tester_1-ff00_0_111", "bash", "-c", + "sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -duration 60s " + + "-remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345", + ) + print(result) + + # Ping again. + loss = self._run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", + count=3,size=1000,interval="1s") + print(f"final ping loss is {loss}") + if loss > 90.0: + print(f"The initial ping command has too high a loss ratio: {loss}") + sys.exit(1) + # Measure BR-111 after the load increase: + metrics_after = measure_br("http://172.20.0.26:30442/metrics") + bfd_changes = metrics_after["router_bfd_state_changes"]["total"] -\ + metrics_before["router_bfd_state_changes"]["total"] + print(f"BFD state changes: {bfd_changes}") + if bfd_changes != 0: + print(f"BFD state should have not changed, but had {bfd_changes} changes.") + sys.exit(1) + busy_fwd = metrics_after["router_dropped_pkts"]["reason"]["busy_forwarder"] -\ + metrics_before["router_dropped_pkts"]["reason"]["busy_forwarder"] + if busy_fwd == 0: + print(f"Insufficient load: no packet drop occurred.") + sys.exit(1) + print(f"router metrics follow.\n" + f"Before:\n-----8<-----\n{metrics_before}\n-----8<-----\n" + f"After: \n-----8<-----\n{metrics_after}\n-----8<-----") + print("Success.") + print(f"-------------------- finished router_priority test") + + def _get_service_info(self, service: str) -> Dict: + """Returns a dictionary with some information about the container. + This function additionally sets the PID of the container to the returned information.""" + output = self.dc("ps", "--format", "json") + # Currently, `docker compose top --format json` does NOT return a valid json string, but a + # bunch of lines containing valid json strings. Split into lines and treat them separately. + output = output.splitlines() + service_dict = None + for line in output: + ps_out = json.loads(line) + if ps_out["Service"] == service: + service_dict = ps_out + break + # If we found the service, get its PID and add it to the dictionary. + if service_dict is not None: + try: + pid = subprocess.check_output( + ["docker", "inspect", "-f", "{{.State.Pid}}", service_dict["Name"]], + text=True, + stderr=subprocess.PIPE, + ).strip() + except subprocess.CalledProcessError as e: + if "No such object" in e.stderr: + raise RuntimeError(f"Container '{service_dict["Name"]}' does not exist") from e + raise RuntimeError("Docker inspect failed") from e + service_dict["PID"] = pid + return service_dict + + def _run_scion_ping(self, src_container:str, dst_endpoint:str, + count:int, size:int, interval:str) -> float: + """Returns the loss rate 0..100""" + cmd = ["scion","ping","--format", "yaml", + "-c", str(count), "-s", str(size), "--interval", str(interval), dst_endpoint] + lines = self.dc("exec", src_container, *cmd) + ping = yaml.safe_load(lines) + stats = ping["statistics"] + return float(stats["packet_loss"]) if __name__ == "__main__": - # base.main(Test) - deleteme() - # sudo tc qdisc add dev vethXYZ root tbf rate 1mbit burst 32kbit latency 400ms - - -# We need the following pip extra packages: -# - prometheus-client -# - requests - - -# docker compose -f gen/scion-dc.yml restart br1-ff00_0_111-1 -# OR -# scion.sh stop ; make && make docker-images && ./scion.sh start && sleep 10 && ./bin/end2end_integration -d - -# ip -o link show | grep scn_000 # BR111->BR110 -# sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-br1-ff00_0_111-1-1 ) -n ip address -# sudo tc qdisc add dev vethfb8dc0d root tbf rate 1mbit burst 32kbit latency 400ms - - -# docker compose -f gen/scion-dc.yml exec -it tester_1-ff00_0_111 bash - -# sudo tcpdump -i any 'udp' -w sender.pcap ; sudo chown juan:juan sender.pcap - - -# docker compose -f gen/scion-dc.yml logs br1-ff00_0_111-1 -f -# curl -s http://172.20.0.26:30442/metrics | grep bfd_sent -# curl -s http://172.20.0.26:30442/metrics | grep bfd -# curl -s http://172.20.0.26:30442/metrics | grep drop -# curl -s http://172.20.0.26:30442/metrics | grep processed_pkts - -# Blast the router with SCION UDP: -# go build -o sender ./demo/router_priority/sender/ && sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' scion-tester_1-ff00_0_111-1 ) -n ./sender -daemon 172.20.0.28:30255 -local 172.20.0.29:0 -remote 1-ff00:0:112,[fd00:f00d:cafe::7f00:15]:12345 -duration 60s - - -# Check that still works after the blast: -# docker compose -f gen/scion-dc.yml exec tester_1-ff00_0_111 scion ping -c 3 1-ff00:0:112,fd00:f00d:cafe::7f00:15 - - -""" -172.20.0.26 BR-1 @ 111 -172.20.0.27 disp CS @ 111 -172.20.0.28 daemon @ 111 -172.20.0.29 tester @ 111 - -ip.addr == 172.20.0.0/16 and ip.src==172.20.0.3 -udp && scion && scion.next_hdr == 202 && scmp.type -udp && scion && scion.src_host == "172.20.0.29" && scion.payload_len == 1108 && scion_udp.dst_port == 12345 - -""" + base.main(Test) diff --git a/tools/env/pip3/requirements.in b/tools/env/pip3/requirements.in index 5f7fd13eef..0a073e7ac2 100644 --- a/tools/env/pip3/requirements.in +++ b/tools/env/pip3/requirements.in @@ -3,5 +3,7 @@ pyyaml==6.0.1 plumbum==1.6.9 supervisor==4.2.5 supervisor-wildcards==0.1.3 +prometheus-client==0.24.1 +requests==2.32.5 # use latest SIX six==1.15.0 diff --git a/tools/env/pip3/requirements.txt b/tools/env/pip3/requirements.txt index a9bc5bf05c..1283484305 100644 --- a/tools/env/pip3/requirements.txt +++ b/tools/env/pip3/requirements.txt @@ -4,10 +4,137 @@ # # bazel run //tools/env/pip3:requirements.update # +certifi==2026.1.4 \ + --hash=sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c \ + --hash=sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120 + # via requests +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 + # via requests +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 + # via requests plumbum==1.6.9 \ --hash=sha256:16b9e19d96c80f2e9d051ef5f04927b834a6ac0ce5d2768eb8662b5cd53e43df \ --hash=sha256:91418dcc66b58ab9d2e3b04b3d1e0d787dc45923154fb8b4a826bd9316dba0d6 # via -r tools/env/pip3/requirements.in +prometheus-client==0.24.1 \ + --hash=sha256:150db128af71a5c2482b36e588fc8a6b95e498750da4b17065947c16070f4055 \ + --hash=sha256:7e0ced7fbbd40f7b84962d5d2ab6f17ef88a72504dcf7c0b40737b43b2a461f9 + # via -r tools/env/pip3/requirements.in pyyaml==6.0.1 \ --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ @@ -50,6 +177,10 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via -r tools/env/pip3/requirements.in +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf + # via -r tools/env/pip3/requirements.in six==1.15.0 \ --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced @@ -65,6 +196,10 @@ toml==0.10.2 \ --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f # via -r tools/env/pip3/requirements.in +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 + # via requests # The following packages are considered to be unsafe in a requirements file: setuptools==69.1.0 \ From 7bfb5f8a08277a71f24f6c7cb0b36e13d0bfb958 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Mon, 2 Feb 2026 15:54:30 +0100 Subject: [PATCH 13/16] Simplify test by running tc in a container. --- demo/router_priority/BUILD.bazel | 1 + demo/router_priority/tc_setup.sh | 12 +++ demo/router_priority/test.py | 151 +++++-------------------------- 3 files changed, 34 insertions(+), 130 deletions(-) create mode 100755 demo/router_priority/tc_setup.sh diff --git a/demo/router_priority/BUILD.bazel b/demo/router_priority/BUILD.bazel index 0f1c7304f7..62f0ebab2b 100644 --- a/demo/router_priority/BUILD.bazel +++ b/demo/router_priority/BUILD.bazel @@ -13,6 +13,7 @@ topogen_test( "--executable=sender:$(location //demo/router_priority/sender)", ], data = [ + "tc_setup.sh", "//demo/router_priority/sender", ], topo = "//topology:tiny.topo", diff --git a/demo/router_priority/tc_setup.sh b/demo/router_priority/tc_setup.sh new file mode 100755 index 0000000000..730ee9c86d --- /dev/null +++ b/demo/router_priority/tc_setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -ex + +NETWORK=$1 +RATE=$2 + +veths=$(bridge link show | awk "/$NETWORK/{print \$2}") +for veth in $veths +do + echo $veth + tc qdisc add dev $veth root tbf rate $RATE latency 1ms burst 50kb mtu 10000 +done diff --git a/demo/router_priority/test.py b/demo/router_priority/test.py index 0f797efb24..26837a3e66 100755 --- a/demo/router_priority/test.py +++ b/demo/router_priority/test.py @@ -19,6 +19,7 @@ from prometheus_client.parser import text_string_to_metric_families from typing import Iterable, Dict, List, Tuple import json +import os import re import requests import subprocess @@ -28,97 +29,6 @@ from plumbum import local -def get_interface_indices(service_info:Dict) -> List[int]: - """ - Returns the equivalent of running: - sudo nsenter -t $(docker inspect -f '{{.State.Pid}}' container ) -n ip link - - :param container: Container name, e.g. scion-br1-ff00_0_111-1-1 - """ - # With nsenter, find the interfaces of the container. - proc = subprocess.run( - ["sudo", "nsenter", "-t", service_info["PID"], "-n", "ip", "link"], - text=True, - capture_output=True, - check=True - ) - # Parse output: the output contains 2 lines per interface, like this: (first line broken at \) - # 2: eth0@if751: mtu 1500 \ - # qdisc noqueue state UP mode DEFAULT group default - # link/ether 0e:40:6f:f3:6e:d7 brd ff:ff:ff:ff:ff:ff link-netnsid 0 - # We only need the first line, and only the second "field" of that line. - iface_re = re.compile(r'^\s*\d+:\s+([^:]+):') - interfaces = [ - m.group(1) - for line in proc.stdout.splitlines() - if (m := iface_re.match(line)) - ] - # Strip eth0@if out of the name of the interface: - index_re = re.compile(r'^eth\d+@if(\d+)') - indices = [ - int(m.group(1)) - for iface in interfaces - if (m:= index_re.match(iface)) - ] - return indices - - -def get_host_bridge_interface(indices: Iterable[int], network_name:str) -> str: - """Given the indices, it returns the first interface that matches the given network.""" - proc = subprocess.run( - ["ip", "-o", "link", "show"], - text=True, - capture_output=True, - check=True - ) - lines = proc.stdout.splitlines() - # Output is like: (lines intentionally broken with "\") - # 869: scn_000: mtu 1500 qdisc noqueue state UP mode \ - # DEFAULT group default \ link/ether 62:53:60:c8:e2:e5 brd ff:ff:ff:ff:ff:ff - # 870: scn_003: mtu 1500 qdisc noqueue state UP mode \ - # DEFAULT group default \ link/ether c2:b7:65:8b:c1:ef brd ff:ff:ff:ff:ff:ff - # 871: vethf9d48a7@if2: mtu 1500 qdisc noqueue \ - # master scn_003 state UP mode DEFAULT group default \ link/ether 86:e9:1d:77:c9:1f brd ff:ff:ff:ff:ff:ff link-netnsid 0 - bridges_re = re.compile(r"^\s*(\d+):\s+([^\s@]+)@if\d+.*master\s+(\S+)") - for line in lines: - m = bridges_re.match(line) - if not m: - continue - idx = int(m.group(1)) - if not idx in indices: - continue - network = m.group(3) - if network == network_name: - return m.group(2) - raise RuntimeError(f"did not find the bridge for {network_name} and indices: {indices}") - - -def set_tc_limits(bridge:str, rate:str, burst:str, latency:str) -> None: - # Try to reset the qdisc of the device. - try: - subprocess.run( - ["sudo", "tc", "qdisc", "del", "dev", bridge, "root"], - check=True, - capture_output=True, - ) - except subprocess.CalledProcessError as e: - # Ignore failure (fails if not already set). - pass - - # Set qdisc of the device to 1Mbps: - try: - subprocess.run( - ["sudo", "tc", "qdisc", "add", "dev", bridge, - "root", "tbf", "rate", rate, "burst", burst, "latency", latency], - text=True, - capture_output=True, - check=True - ) - except subprocess.CalledProcessError as e: - print(e.stderr) - raise RuntimeError(f"command tc failed: {e.stderr}") from e - - def measure_br(url: str): metrics = { "router_bfd_state_changes":{ @@ -160,6 +70,26 @@ def measure_br(url: str): class Test(base.TestTopogen): def setup_prepare(self): super().setup_prepare() + # Add throttling to the BR-111 <-> BR-110 link. + scion_dc = self.artifacts / "gen/scion-dc.yml" + with open(scion_dc, "r") as file: + dc = yaml.load(file, Loader=yaml.FullLoader) + dc["services"]["tc_setup"] = { + "image": "scion/tester:latest", + "cap_add": ["NET_ADMIN"], + "volumes": [{ + "type": "bind", + "source": os.path.realpath("demo/router_priority/tc_setup.sh"), + "target": "/share/tc_setup.sh", + }], + "entrypoint": ["/bin/bash", "-exc", + "ls -l /share; /share/tc_setup.sh scn_000 512kbit; " + "echo TC limits applied to scn_000"], + "depends_on": ["br1-ff00_0_110-1", "br1-ff00_0_111-1"], + "network_mode": "host", + } + with open(scion_dc, "w") as file: + yaml.dump(dc, file) def _run(self): print("-------------------- running router_priority test") @@ -168,17 +98,6 @@ def _run(self): sender_bin = local["realpath"](self.get_executable("sender").executable).strip() self.dc("cp", sender_bin, "tester_1-ff00_0_111" + ":/bin/") - # Get the BR-111 service information. - service_info = self._get_service_info("br1-ff00_0_111-1") - # List interfaces of the BR-111 container. - indices = get_interface_indices(service_info) - # Match those interfaces with one of the host interfaces. - bridge = get_host_bridge_interface(indices, "scn_000") - print(f"bridge is: {bridge}") - # Limit the BR-111 -> BR-110 interface to 1Mbps. - set_tc_limits(bridge, rate="512kbit", burst="32kbit", latency="400ms") - print(f"tc limits applied to host interface {bridge} (scn_000: BR-110 <-> BR-111)") - # Measure ping loss before loading the BR: loss = self._run_scion_ping("tester_1-ff00_0_111", "1-ff00:0:112,fd00:f00d:cafe::7f00:15", count=3,size=1000,interval="1s") @@ -222,34 +141,6 @@ def _run(self): print("Success.") print(f"-------------------- finished router_priority test") - def _get_service_info(self, service: str) -> Dict: - """Returns a dictionary with some information about the container. - This function additionally sets the PID of the container to the returned information.""" - output = self.dc("ps", "--format", "json") - # Currently, `docker compose top --format json` does NOT return a valid json string, but a - # bunch of lines containing valid json strings. Split into lines and treat them separately. - output = output.splitlines() - service_dict = None - for line in output: - ps_out = json.loads(line) - if ps_out["Service"] == service: - service_dict = ps_out - break - # If we found the service, get its PID and add it to the dictionary. - if service_dict is not None: - try: - pid = subprocess.check_output( - ["docker", "inspect", "-f", "{{.State.Pid}}", service_dict["Name"]], - text=True, - stderr=subprocess.PIPE, - ).strip() - except subprocess.CalledProcessError as e: - if "No such object" in e.stderr: - raise RuntimeError(f"Container '{service_dict["Name"]}' does not exist") from e - raise RuntimeError("Docker inspect failed") from e - service_dict["PID"] = pid - return service_dict - def _run_scion_ping(self, src_container:str, dst_endpoint:str, count:int, size:int, interval:str) -> float: """Returns the loss rate 0..100""" From 80fd6cdb91c2e0d7f8cf4437f373ec54451fc15a Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Tue, 3 Feb 2026 15:38:36 +0100 Subject: [PATCH 14/16] Project config changes after rebase. --- demo/router_priority/BUILD.bazel | 8 ++++---- demo/router_priority/sender/BUILD.bazel | 1 + demo/router_priority/sender/main.go | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/demo/router_priority/BUILD.bazel b/demo/router_priority/BUILD.bazel index 62f0ebab2b..23b20ce045 100644 --- a/demo/router_priority/BUILD.bazel +++ b/demo/router_priority/BUILD.bazel @@ -5,10 +5,6 @@ load("//acceptance/common:topogen.bzl", "topogen_test") topogen_test( name = "test", src = "test.py", - deps = [ - requirement("prometheus-client"), - requirement("requests"), - ], args = [ "--executable=sender:$(location //demo/router_priority/sender)", ], @@ -17,4 +13,8 @@ topogen_test( "//demo/router_priority/sender", ], topo = "//topology:tiny.topo", + deps = [ + requirement("prometheus-client"), + requirement("requests"), + ], ) diff --git a/demo/router_priority/sender/BUILD.bazel b/demo/router_priority/sender/BUILD.bazel index 7f3e9814d3..0255c4a038 100644 --- a/demo/router_priority/sender/BUILD.bazel +++ b/demo/router_priority/sender/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//pkg/daemon:go_default_library", + "//pkg/daemon/types:go_default_library", "//pkg/snet:go_default_library", "//pkg/snet/metrics:go_default_library", ], diff --git a/demo/router_priority/sender/main.go b/demo/router_priority/sender/main.go index c7eaf2ab25..b1745ba9d8 100644 --- a/demo/router_priority/sender/main.go +++ b/demo/router_priority/sender/main.go @@ -19,6 +19,7 @@ import ( "time" "github.com/scionproto/scion/pkg/daemon" + daemontypes "github.com/scionproto/scion/pkg/daemon/types" "github.com/scionproto/scion/pkg/snet" "github.com/scionproto/scion/pkg/snet/metrics" ) @@ -73,7 +74,7 @@ func main() { } // Get paths. - paths, err := daemonConn.Paths(ctx, remote.IA, localIA, daemon.PathReqFlags{}) + paths, err := daemonConn.Paths(ctx, remote.IA, localIA, daemontypes.PathReqFlags{}) panicOnError(err) if len(paths) == 0 { panic("no paths") From 4a10822704694c23fdbbb74a0ccd0faa3b1e7b37 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Tue, 3 Feb 2026 15:38:52 +0100 Subject: [PATCH 15/16] Update the module lock file after rebase. --- MODULE.bazel.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 58f20cac0e..40ea6b08f7 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1311,7 +1311,7 @@ "usagesDigest": "xr+U7navw2+SHogogmHRboKLbXAnkZfsctPQuyjmArw=", "recordedFileInputs": { "@@//doc/requirements.txt": "69647040f4c4bdd93fd8369b245316b08cfabd17a23693d833081b5785c0f131", - "@@//tools/env/pip3/requirements.txt": "92aa5b99f8051e7e2528f5ff44bb2cb263e3da1682de73908e07e173c2a417ac", + "@@//tools/env/pip3/requirements.txt": "d2f5e829afd46ab2db03085eebc6f2271a42ab5cc46a4d0973ba2c9a537862b9", "@@//tools/lint/python/requirements.txt": "c6eb43e931b4200ae71e62fc65bc78d72224495a1648dbc0a565f09571724bd8", "@@rules_fuzzing+//fuzzing/requirements.txt": "ab04664be026b632a0d2a2446c4f65982b7654f5b6851d2f9d399a19b7242a5b", "@@rules_python+//tools/publish/requirements_darwin.txt": "095d4a4f3d639dce831cd493367631cd51b53665292ab20194bac2c0c6458fa8", From fa11d3f48e335c42ae76644a90d074aa2a22c978 Mon Sep 17 00:00:00 2001 From: "Juan A. Garcia Pardo" Date: Mon, 9 Feb 2026 07:52:25 +0100 Subject: [PATCH 16/16] Move router_priority from demo to acceptance. --- {demo => acceptance}/router_priority/BUILD.bazel | 4 ++-- {demo => acceptance}/router_priority/README.md | 4 ++-- {demo => acceptance}/router_priority/sender/BUILD.bazel | 2 +- {demo => acceptance}/router_priority/sender/main.go | 0 {demo => acceptance}/router_priority/tc_setup.sh | 0 {demo => acceptance}/router_priority/test.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename {demo => acceptance}/router_priority/BUILD.bazel (76%) rename {demo => acceptance}/router_priority/README.md (98%) rename {demo => acceptance}/router_priority/sender/BUILD.bazel (85%) rename {demo => acceptance}/router_priority/sender/main.go (100%) rename {demo => acceptance}/router_priority/tc_setup.sh (100%) rename {demo => acceptance}/router_priority/test.py (98%) diff --git a/demo/router_priority/BUILD.bazel b/acceptance/router_priority/BUILD.bazel similarity index 76% rename from demo/router_priority/BUILD.bazel rename to acceptance/router_priority/BUILD.bazel index 23b20ce045..9106a5d3b8 100644 --- a/demo/router_priority/BUILD.bazel +++ b/acceptance/router_priority/BUILD.bazel @@ -6,11 +6,11 @@ topogen_test( name = "test", src = "test.py", args = [ - "--executable=sender:$(location //demo/router_priority/sender)", + "--executable=sender:$(location //acceptance/router_priority/sender)", ], data = [ "tc_setup.sh", - "//demo/router_priority/sender", + "//acceptance/router_priority/sender", ], topo = "//topology:tiny.topo", deps = [ diff --git a/demo/router_priority/README.md b/acceptance/router_priority/README.md similarity index 98% rename from demo/router_priority/README.md rename to acceptance/router_priority/README.md index 7f4dcb82f1..d74427c64b 100644 --- a/demo/router_priority/README.md +++ b/acceptance/router_priority/README.md @@ -80,7 +80,7 @@ without any kind of flow control. The regular `scion ping` does not work here, as it paces itself if responses are not received. -## How to run the demo test +## How to run the test independently 1. [Set up the development environment](https://docs.scion.org/en/latest/build/setup.html) -2. `bazel test --test_output=streamed --cache_test_results=no //demo/router_priority:test` +2. `bazel test --test_output=streamed --cache_test_results=no //acceptance/router_priority:test` diff --git a/demo/router_priority/sender/BUILD.bazel b/acceptance/router_priority/sender/BUILD.bazel similarity index 85% rename from demo/router_priority/sender/BUILD.bazel rename to acceptance/router_priority/sender/BUILD.bazel index 0255c4a038..d5bcd606b4 100644 --- a/demo/router_priority/sender/BUILD.bazel +++ b/acceptance/router_priority/sender/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", srcs = ["main.go"], - importpath = "github.com/scionproto/scion/demo/router_priority/sender", + importpath = "github.com/scionproto/scion/acceptance/router_priority/sender", visibility = ["//visibility:private"], deps = [ "//pkg/daemon:go_default_library", diff --git a/demo/router_priority/sender/main.go b/acceptance/router_priority/sender/main.go similarity index 100% rename from demo/router_priority/sender/main.go rename to acceptance/router_priority/sender/main.go diff --git a/demo/router_priority/tc_setup.sh b/acceptance/router_priority/tc_setup.sh similarity index 100% rename from demo/router_priority/tc_setup.sh rename to acceptance/router_priority/tc_setup.sh diff --git a/demo/router_priority/test.py b/acceptance/router_priority/test.py similarity index 98% rename from demo/router_priority/test.py rename to acceptance/router_priority/test.py index 26837a3e66..a7eaa83d75 100755 --- a/demo/router_priority/test.py +++ b/acceptance/router_priority/test.py @@ -79,7 +79,7 @@ def setup_prepare(self): "cap_add": ["NET_ADMIN"], "volumes": [{ "type": "bind", - "source": os.path.realpath("demo/router_priority/tc_setup.sh"), + "source": os.path.realpath("acceptance/router_priority/tc_setup.sh"), "target": "/share/tc_setup.sh", }], "entrypoint": ["/bin/bash", "-exc",