diff --git a/cmd/metal-api/internal/datastore/integer.go b/cmd/metal-api/internal/datastore/integer.go index 04a57a4fa..a5784dc4e 100644 --- a/cmd/metal-api/internal/datastore/integer.go +++ b/cmd/metal-api/internal/datastore/integer.go @@ -64,9 +64,12 @@ func (rs *RethinkStore) GetIntegerPool(pool IntegerPoolType) (*IntegerPool, erro // initIntegerPool initializes a pool to acquire unique integers from. // the acquired integers are used from the network service for defining the: +// one integer for: // - vrf name // - vni // - vxlan-id +// and one integer for: +// - asn-id offset added to 4210000000 (ASNBase) // // the integer range has a vxlan-id constraint from Cumulus: // net add vxlan vxlan10 vxlan id @@ -74,6 +77,7 @@ func (rs *RethinkStore) GetIntegerPool(pool IntegerPoolType) (*IntegerPool, erro // // in order not to impact performance too much, we limited the range of integers to 2^17=131072, // which includes the range that we typically used for vrf names in the past. +// By this limitation we limit the number of machines possible to manage to ~130.000 ! // // the implementation of the pool works as follows: // - write the given range of integers to the rethinkdb on first start (with the integer as the document id) diff --git a/cmd/metal-api/internal/metal/ip.go b/cmd/metal-api/internal/metal/ip.go index 461604832..8533c7a3f 100644 --- a/cmd/metal-api/internal/metal/ip.go +++ b/cmd/metal-api/internal/metal/ip.go @@ -2,13 +2,12 @@ package metal import ( "fmt" - "github.com/metal-stack/metal-lib/pkg/tag" - "net" "strings" "time" + "github.com/metal-stack/metal-lib/pkg/tag" + "github.com/metal-stack/metal-api/cmd/metal-api/internal/tags" - "github.com/pkg/errors" ) // IPType is the type of an ip. @@ -78,22 +77,6 @@ func (ip *IP) SetCreated(created time.Time) { ip.Created = created } -// ASNBase is the offset for all Machine ASN´s -const ASNBase = int64(4200000000) - -// ASN calculate a ASN from the ip -// we start to calculate ASNs for machines with the first ASN in the 32bit ASN range and -// add the last 2 octets of the ip of the machine to achieve unique ASNs per vrf -// TODO consider using IntegerPool here as well to calculate the addition to ASNBase -func (ip *IP) ASN() (int64, error) { - i := net.ParseIP(ip.IPAddress) - if i == nil { - return int64(-1), errors.Errorf("unable to parse ip %s", ip.IPAddress) - } - asn := ASNBase + int64(i[14])*256 + int64(i[15]) - return asn, nil -} - // GetScope determines the scope of an ip address func (ip *IP) GetScope() IPScope { if ip.ProjectID == "" { diff --git a/cmd/metal-api/internal/metal/ip_test.go b/cmd/metal-api/internal/metal/ip_test.go index 9ad30c0cc..dcbd0a3da 100644 --- a/cmd/metal-api/internal/metal/ip_test.go +++ b/cmd/metal-api/internal/metal/ip_test.go @@ -1,9 +1,10 @@ package metal import ( - "github.com/metal-stack/metal-lib/pkg/tag" "testing" + "github.com/metal-stack/metal-lib/pkg/tag" + "github.com/google/go-cmp/cmp" ) @@ -124,18 +125,3 @@ func TestGetScope(t *testing.T) { }) } } - -func TestIPToASN(t *testing.T) { - ipaddress := IP{ - IPAddress: "10.0.1.2", - } - - asn, err := ipaddress.ASN() - if err != nil { - t.Errorf("no error expected got:%v", err) - } - - if asn != 4200000258 { - t.Errorf("expected 4200000258 got: %d", asn) - } -} diff --git a/cmd/metal-api/internal/metal/machine.go b/cmd/metal-api/internal/metal/machine.go index cbfee2dfc..d6da342b6 100644 --- a/cmd/metal-api/internal/metal/machine.go +++ b/cmd/metal-api/internal/metal/machine.go @@ -158,7 +158,7 @@ type MachineNetwork struct { DestinationPrefixes []string `rethinkdb:"destinationprefixes" json:"destinationprefixes"` Vrf uint `rethinkdb:"vrf" json:"vrf"` Private bool `rethinkdb:"private" json:"private"` - ASN int64 `rethinkdb:"asn" json:"asn"` + ASN uint32 `rethinkdb:"asn" json:"asn"` Nat bool `rethinkdb:"nat" json:"nat"` Underlay bool `rethinkdb:"underlay" json:"underlay"` } diff --git a/cmd/metal-api/internal/service/asn.go b/cmd/metal-api/internal/service/asn.go new file mode 100644 index 000000000..833ecea9e --- /dev/null +++ b/cmd/metal-api/internal/service/asn.go @@ -0,0 +1,51 @@ +package service + +import ( + "fmt" + + "github.com/metal-stack/metal-api/cmd/metal-api/internal/datastore" +) + +const ( + // ASNMin is the minimum asn defined according to + // https://en.wikipedia.org/wiki/Autonomous_system_(Internet) + ASNMin = uint32(4200000000) + + // ASNBase is the offset for all Machine ASN´s + ASNBase = uint32(4210000000) + + // ASNMax defines the maximum allowed asn + // https://en.wikipedia.org/wiki/Autonomous_system_(Internet) + ASNMax = uint32(4294967294) +) + +// acquireASN fetches a unique integer by using the existing integer pool and adding to ASNBase +func acquireASN(ds *datastore.RethinkStore) (*uint32, error) { + asnPool, err := ds.GetIntegerPool(datastore.ASNIntegerPool) + if err != nil { + return nil, err + } + i, err := asnPool.AcquireRandomUniqueInteger() + if err != nil { + return nil, err + } + asn := ASNBase + uint32(i) + if asn > ASNMax { + return nil, fmt.Errorf("unable to calculate asn, got a asn larger than ASNMax: %d > %d", asn, ASNMax) + } + return &asn, nil +} + +// releaseASN will release the asn from the integerpool +func releaseASN(ds *datastore.RethinkStore, asn uint32) error { + if asn < ASNBase || asn > ASNMax { + return fmt.Errorf("asn %d might not be smaller than:%d or larger than %d", asn, ASNBase, ASNMax) + } + i := uint(asn - ASNBase) + + asnPool, err := ds.GetIntegerPool(datastore.ASNIntegerPool) + if err != nil { + return err + } + return asnPool.ReleaseUniqueInteger(i) +} diff --git a/cmd/metal-api/internal/service/async-actor.go b/cmd/metal-api/internal/service/async-actor.go index 9fad8c133..e07fa2c41 100644 --- a/cmd/metal-api/internal/service/async-actor.go +++ b/cmd/metal-api/internal/service/async-actor.go @@ -81,6 +81,7 @@ func (a *asyncActor) releaseMachineNetworks(machine *metal.Machine) error { if machine.Allocation == nil { return nil } + var asn uint32 for _, machineNetwork := range machine.Allocation.MachineNetworks { if machineNetwork.IPs == nil { continue @@ -101,6 +102,18 @@ func (a *asyncActor) releaseMachineNetworks(machine *metal.Machine) error { return err } } + // all machineNetworks have the same ASN, must only be released once + // in the old way asn was in the range of 4200000000 + offset from last two octets of private ip + // but we must only release asn which are acquired from 4210000000 and above from the ASNIntegerPool + if machineNetwork.ASN >= ASNBase { + asn = machineNetwork.ASN + } + } + if asn >= ASNBase { + err := releaseASN(a.RethinkStore, asn) + if err != nil { + return err + } } // it can happen that an IP gets properly allocated for a machine but @@ -119,7 +132,6 @@ func (a *asyncActor) releaseMachineNetworks(machine *metal.Machine) error { return err } } - return nil } diff --git a/cmd/metal-api/internal/service/machine-service.go b/cmd/metal-api/internal/service/machine-service.go index abd8fbcd2..4f9b6a291 100644 --- a/cmd/metal-api/internal/service/machine-service.go +++ b/cmd/metal-api/internal/service/machine-service.go @@ -3,14 +3,15 @@ package service import ( "context" "fmt" - "github.com/metal-stack/metal-api/cmd/metal-api/internal/grpc" - "github.com/pkg/errors" "net" "net/http" "strconv" "strings" "time" + "github.com/metal-stack/metal-api/cmd/metal-api/internal/grpc" + "github.com/pkg/errors" + "golang.org/x/crypto/ssh" "github.com/metal-stack/metal-lib/httperrors" @@ -1039,12 +1040,12 @@ func makeNetworks(ds *datastore.RethinkStore, ipamer ipam.IPAMer, allocationSpec } // the metal-networker expects to have the same unique ASN on all networks of this machine - asn, err := makeASN(networks) + asn, err := acquireASN(ds) if err != nil { return err } for _, n := range alloc.MachineNetworks { - n.ASN = asn + n.ASN = *asn } return nil @@ -1261,26 +1262,6 @@ func makeMachineNetwork(ds *datastore.RethinkStore, ipamer ipam.IPAMer, allocati return &machineNetwork, nil } -// makeASN we can use the IP of the private network (which always have to be present and unique) -// for generating a unique ASN. -func makeASN(networks allocationNetworkMap) (int64, error) { - privateNetwork, err := getPrivateNetwork(networks) - if err != nil { - return 0, err - } - - if len(privateNetwork.ips) == 0 { - return 0, fmt.Errorf("private network has no IPs, which would result in a machine without an IP") - } - - asn, err := privateNetwork.ips[0].ASN() - if err != nil { - return 0, err - } - - return asn, nil -} - // makeMachineTags constructs the tags of the machine. // following tags are added in the following precedence (from lowest to highest in case of duplication): // - external network labels (concatenated, from all machine networks that this machine belongs to) @@ -1340,7 +1321,7 @@ func makeMachineSystemLabels(m *metal.Machine) map[string]string { for _, n := range m.Allocation.MachineNetworks { if n.Private { if n.ASN != 0 { - labels[tag.MachineNetworkPrimaryASN] = strconv.FormatInt(n.ASN, 10) + labels[tag.MachineNetworkPrimaryASN] = strconv.FormatInt(int64(n.ASN), 10) break } } diff --git a/cmd/metal-api/internal/service/network-service.go b/cmd/metal-api/internal/service/network-service.go index ec58efead..69b2ac1a0 100644 --- a/cmd/metal-api/internal/service/network-service.go +++ b/cmd/metal-api/internal/service/network-service.go @@ -336,13 +336,7 @@ func (r networkResource) createNetwork(request *restful.Request, response *restf } if vrf != 0 { - vrfPool, err := r.ds.GetIntegerPool(datastore.VRFIntegerPool) - if err != nil { - if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not acquire vrf: %v", err)) { - return - } - } - _, err = vrfPool.AcquireUniqueInteger(vrf) + err = acquireVRF(r.ds, vrf) if err != nil { if !metal.IsConflict(err) { if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not acquire vrf: %v", err)) { @@ -469,11 +463,7 @@ func (r networkResource) allocateNetwork(request *restful.Request, response *res } func createChildNetwork(ds *datastore.RethinkStore, ipamer ipam.IPAMer, nwSpec *metal.Network, parent *metal.Network, childLength int) (*metal.Network, error) { - vrfPool, err := ds.GetIntegerPool(datastore.VRFIntegerPool) - if err != nil { - return nil, err - } - vrf, err := vrfPool.AcquireRandomUniqueInteger() + vrf, err := acquireRandomVRF(ds) if err != nil { return nil, fmt.Errorf("Could not acquire a vrf: %v", err) } @@ -499,7 +489,7 @@ func createChildNetwork(ds *datastore.RethinkStore, ipamer ipam.IPAMer, nwSpec * Nat: parent.Nat, PrivateSuper: false, Underlay: false, - Vrf: vrf, + Vrf: *vrf, ParentNetworkID: parent.ID, Labels: nwSpec.Labels, } @@ -540,13 +530,7 @@ func (r networkResource) freeNetwork(request *restful.Request, response *restful } if nw.Vrf != 0 { - vrfPool, err := r.ds.GetIntegerPool(datastore.VRFIntegerPool) - if err != nil { - if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not release vrf: %v", err)) { - return - } - } - err = vrfPool.ReleaseUniqueInteger(nw.Vrf) + err = releaseVRF(r.ds, nw.Vrf) if err != nil { if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not release vrf: %v", err)) { return @@ -689,13 +673,7 @@ func (r networkResource) deleteNetwork(request *restful.Request, response *restf } if nw.Vrf != 0 { - vrfPool, err := r.ds.GetIntegerPool(datastore.VRFIntegerPool) - if err != nil { - if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not release vrf: %v", err)) { - return - } - } - err = vrfPool.ReleaseUniqueInteger(nw.Vrf) + err = releaseVRF(r.ds, nw.Vrf) if err != nil { if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not release vrf: %v", err)) { return diff --git a/cmd/metal-api/internal/service/v1/machine.go b/cmd/metal-api/internal/service/v1/machine.go index 5d1cb9c5b..fa452436b 100644 --- a/cmd/metal-api/internal/service/v1/machine.go +++ b/cmd/metal-api/internal/service/v1/machine.go @@ -51,10 +51,11 @@ type BootInfo struct { } type MachineNetwork struct { - NetworkID string `json:"networkid" description:"the networkID of the allocated machine in this vrf"` - Prefixes []string `json:"prefixes" description:"the prefixes of this network"` - IPs []string `json:"ips" description:"the ip addresses of the allocated machine in this vrf"` - Vrf uint `json:"vrf" description:"the vrf of the allocated machine"` + NetworkID string `json:"networkid" description:"the networkID of the allocated machine in this vrf"` + Prefixes []string `json:"prefixes" description:"the prefixes of this network"` + IPs []string `json:"ips" description:"the ip addresses of the allocated machine in this vrf"` + Vrf uint `json:"vrf" description:"the vrf of the allocated machine"` + // Attention, uint32 is converted to integer by swagger which is int32 which is to small to hold a asn ASN int64 `json:"asn" description:"ASN number for this network in the bgp configuration"` Private bool `json:"private" description:"indicates whether this network is the private network of this machine"` Nat bool `json:"nat" description:"if set to true, packets leaving this network get masqueraded behind interface ip"` @@ -410,7 +411,7 @@ func NewMachineResponse(m *metal.Machine, s *metal.Size, p *metal.Partition, i * NetworkID: nw.NetworkID, IPs: ips, Vrf: nw.Vrf, - ASN: nw.ASN, + ASN: int64(nw.ASN), Private: nw.Private, Nat: nw.Nat, Underlay: nw.Underlay, diff --git a/cmd/metal-api/internal/service/vrf.go b/cmd/metal-api/internal/service/vrf.go new file mode 100644 index 000000000..5ed5397d7 --- /dev/null +++ b/cmd/metal-api/internal/service/vrf.go @@ -0,0 +1,36 @@ +package service + +import ( + "fmt" + + "github.com/metal-stack/metal-api/cmd/metal-api/internal/datastore" +) + +// acquireRandomVRF will grab a unique but random vrf out of the vrfintegerpool +func acquireRandomVRF(ds *datastore.RethinkStore) (*uint, error) { + vrfPool, err := ds.GetIntegerPool(datastore.VRFIntegerPool) + if err != nil { + return nil, fmt.Errorf("could not acquire random vrf: %v", err) + } + vrf, err := vrfPool.AcquireRandomUniqueInteger() + return &vrf, err +} + +// acquireVRF will the given vrf out of the vrfintegerpool if not available a error is thrown +func acquireVRF(ds *datastore.RethinkStore, vrf uint) error { + vrfPool, err := ds.GetIntegerPool(datastore.VRFIntegerPool) + if err != nil { + return fmt.Errorf("could not acquire vrf:%d %v", vrf, err) + } + _, err = vrfPool.AcquireUniqueInteger(vrf) + return err +} + +// releaseVRF will return the given vrf to the vrfintegerpool for reuse +func releaseVRF(ds *datastore.RethinkStore, vrf uint) error { + vrfPool, err := ds.GetIntegerPool(datastore.VRFIntegerPool) + if err != nil { + return fmt.Errorf("could not release vrf: %v", err) + } + return vrfPool.ReleaseUniqueInteger(vrf) +}