Fix asn allocation (#105)

cmd/metal-api/internal/datastore/integer.go

@@ -64,16 +64,20 @@ func (rs *RethinkStore) GetIntegerPool(pool IntegerPoolType) (*IntegerPool, erro
 
 // initIntegerPool initializes a pool to acquire unique integers from.
 // the acquired integers are used from the network service for defining the:
+// one integer for:
 // - vrf name
 // - vni
 // - vxlan-id
+// and one integer for:
+// - asn-id offset added to 4210000000 (ASNBase)
 //
 // the integer range has a vxlan-id constraint from Cumulus:
 // 	net add vxlan vxlan10 vxlan id
 //  <1-16777214>  :  An integer from 1 to 16777214
 //
 // in order not to impact performance too much, we limited the range of integers to 2^17=131072,
 // which includes the range that we typically used for vrf names in the past.
+// By this limitation we limit the number of machines possible to manage to ~130.000 !
 //
 // the implementation of the pool works as follows:
 // - write the given range of integers to the rethinkdb on first start (with the integer as the document id)

cmd/metal-api/internal/metal/ip.go

@@ -2,13 +2,12 @@ package metal
 
 import (
 	"fmt"
-	"github.com/metal-stack/metal-lib/pkg/tag"
-	"net"
 	"strings"
 	"time"
 
+	"github.com/metal-stack/metal-lib/pkg/tag"
+
 	"github.com/metal-stack/metal-api/cmd/metal-api/internal/tags"
-	"github.com/pkg/errors"
 )
 
 // IPType is the type of an ip.
@@ -78,22 +77,6 @@ func (ip *IP) SetCreated(created time.Time) {
 	ip.Created = created
 }
 
-// ASNBase is the offset for all Machine ASN´s
-const ASNBase = int64(4200000000)
-
-// ASN calculate a ASN from the ip
-// we start to calculate ASNs for machines with the first ASN in the 32bit ASN range and
-// add the last 2 octets of the ip of the machine to achieve unique ASNs per vrf
-// TODO consider using IntegerPool here as well to calculate the addition to ASNBase
-func (ip *IP) ASN() (int64, error) {
-	i := net.ParseIP(ip.IPAddress)
-	if i == nil {
-		return int64(-1), errors.Errorf("unable to parse ip %s", ip.IPAddress)
-	}
-	asn := ASNBase + int64(i[14])*256 + int64(i[15])
-	return asn, nil
-}
-
 // GetScope determines the scope of an ip address
 func (ip *IP) GetScope() IPScope {
 	if ip.ProjectID == "" {

cmd/metal-api/internal/metal/ip_test.go

@@ -1,9 +1,10 @@
 package metal
 
 import (
-	"github.com/metal-stack/metal-lib/pkg/tag"
 	"testing"
 
+	"github.com/metal-stack/metal-lib/pkg/tag"
+
 	"github.com/google/go-cmp/cmp"
 )
 
@@ -124,18 +125,3 @@ func TestGetScope(t *testing.T) {
 		})
 	}
 }
-
-func TestIPToASN(t *testing.T) {
-	ipaddress := IP{
-		IPAddress: "10.0.1.2",
-	}
-
-	asn, err := ipaddress.ASN()
-	if err != nil {
-		t.Errorf("no error expected got:%v", err)
-	}
-
-	if asn != 4200000258 {
-		t.Errorf("expected 4200000258 got: %d", asn)
-	}
-}

cmd/metal-api/internal/metal/machine.go

@@ -158,7 +158,7 @@ type MachineNetwork struct {
 	DestinationPrefixes []string `rethinkdb:"destinationprefixes" json:"destinationprefixes"`
 	Vrf                 uint     `rethinkdb:"vrf" json:"vrf"`
 	Private             bool     `rethinkdb:"private" json:"private"`
-	ASN                 int64    `rethinkdb:"asn" json:"asn"`
+	ASN                 uint32   `rethinkdb:"asn" json:"asn"`
 	Nat                 bool     `rethinkdb:"nat" json:"nat"`
 	Underlay            bool     `rethinkdb:"underlay" json:"underlay"`
 }

cmd/metal-api/internal/service/asn.go

@@ -0,0 +1,51 @@
+package service
+
+import (
+	"fmt"
+
+	"github.com/metal-stack/metal-api/cmd/metal-api/internal/datastore"
+)
+
+const (
+	// ASNMin is the minimum asn defined according to
+	// https://en.wikipedia.org/wiki/Autonomous_system_(Internet)
+	ASNMin = uint32(4200000000)
+
+	// ASNBase is the offset for all Machine ASN´s
+	ASNBase = uint32(4210000000)
+
+	// ASNMax defines the maximum allowed asn
+	// https://en.wikipedia.org/wiki/Autonomous_system_(Internet)
+	ASNMax = uint32(4294967294)
+)
+
+// acquireASN fetches a unique integer by using the existing integer pool and adding to ASNBase
+func acquireASN(ds *datastore.RethinkStore) (*uint32, error) {
+	asnPool, err := ds.GetIntegerPool(datastore.ASNIntegerPool)
+	if err != nil {
+		return nil, err
+	}
+	i, err := asnPool.AcquireRandomUniqueInteger()
+	if err != nil {
+		return nil, err
+	}
+	asn := ASNBase + uint32(i)
+	if asn > ASNMax {
+		return nil, fmt.Errorf("unable to calculate asn, got a asn larger than ASNMax: %d > %d", asn, ASNMax)
+	}
+	return &asn, nil
+}
+
+// releaseASN will release the asn from the integerpool
+func releaseASN(ds *datastore.RethinkStore, asn uint32) error {
+	if asn < ASNBase || asn > ASNMax {
+		return fmt.Errorf("asn %d might not be smaller than:%d or larger than %d", asn, ASNBase, ASNMax)
+	}
+	i := uint(asn - ASNBase)
+
+	asnPool, err := ds.GetIntegerPool(datastore.ASNIntegerPool)
+	if err != nil {
+		return err
+	}
+	return asnPool.ReleaseUniqueInteger(i)
+}

cmd/metal-api/internal/service/async-actor.go

@@ -81,6 +81,7 @@ func (a *asyncActor) releaseMachineNetworks(machine *metal.Machine) error {
 	if machine.Allocation == nil {
 		return nil
 	}
+	var asn uint32
 	for _, machineNetwork := range machine.Allocation.MachineNetworks {
 		if machineNetwork.IPs == nil {
 			continue
@@ -101,6 +102,18 @@ func (a *asyncActor) releaseMachineNetworks(machine *metal.Machine) error {
 				return err
 			}
 		}
+		// all machineNetworks have the same ASN, must only be released once
+		// in the old way asn was in the range of 4200000000 + offset from last two octets of private ip
+		// but we must only release asn which are acquired from 4210000000 and above from the ASNIntegerPool
+		if machineNetwork.ASN >= ASNBase {
+			asn = machineNetwork.ASN
+		}
+	}
+	if asn >= ASNBase {
+		err := releaseASN(a.RethinkStore, asn)
+		if err != nil {
+			return err
+		}
 	}
 
 	// it can happen that an IP gets properly allocated for a machine but
@@ -119,7 +132,6 @@ func (a *asyncActor) releaseMachineNetworks(machine *metal.Machine) error {
 			return err
 		}
 	}
-
 	return nil
 }
 

cmd/metal-api/internal/service/machine-service.go

@@ -3,14 +3,15 @@ package service
 import (
 	"context"
 	"fmt"
-	"github.com/metal-stack/metal-api/cmd/metal-api/internal/grpc"
-	"github.com/pkg/errors"
 	"net"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/metal-stack/metal-api/cmd/metal-api/internal/grpc"
+	"github.com/pkg/errors"
+
 	"golang.org/x/crypto/ssh"
 
 	"github.com/metal-stack/metal-lib/httperrors"
@@ -1039,12 +1040,12 @@ func makeNetworks(ds *datastore.RethinkStore, ipamer ipam.IPAMer, allocationSpec
 	}
 
 	// the metal-networker expects to have the same unique ASN on all networks of this machine
-	asn, err := makeASN(networks)
+	asn, err := acquireASN(ds)
 	if err != nil {
 		return err
 	}
 	for _, n := range alloc.MachineNetworks {
-		n.ASN = asn
+		n.ASN = *asn
 	}
 
 	return nil
@@ -1261,26 +1262,6 @@ func makeMachineNetwork(ds *datastore.RethinkStore, ipamer ipam.IPAMer, allocati
 	return &machineNetwork, nil
 }
 
-// makeASN we can use the IP of the private network (which always have to be present and unique)
-// for generating a unique ASN.
-func makeASN(networks allocationNetworkMap) (int64, error) {
-	privateNetwork, err := getPrivateNetwork(networks)
-	if err != nil {
-		return 0, err
-	}
-
-	if len(privateNetwork.ips) == 0 {
-		return 0, fmt.Errorf("private network has no IPs, which would result in a machine without an IP")
-	}
-
-	asn, err := privateNetwork.ips[0].ASN()
-	if err != nil {
-		return 0, err
-	}
-
-	return asn, nil
-}
-
 // makeMachineTags constructs the tags of the machine.
 // following tags are added in the following precedence (from lowest to highest in case of duplication):
 // - external network labels (concatenated, from all machine networks that this machine belongs to)
@@ -1340,7 +1321,7 @@ func makeMachineSystemLabels(m *metal.Machine) map[string]string {
 	for _, n := range m.Allocation.MachineNetworks {
 		if n.Private {
 			if n.ASN != 0 {
-				labels[tag.MachineNetworkPrimaryASN] = strconv.FormatInt(n.ASN, 10)
+				labels[tag.MachineNetworkPrimaryASN] = strconv.FormatInt(int64(n.ASN), 10)
 				break
 			}
 		}

cmd/metal-api/internal/service/network-service.go

@@ -336,13 +336,7 @@ func (r networkResource) createNetwork(request *restful.Request, response *restf
 	}
 
 	if vrf != 0 {
-		vrfPool, err := r.ds.GetIntegerPool(datastore.VRFIntegerPool)
-		if err != nil {
-			if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not acquire vrf: %v", err)) {
-				return
-			}
-		}
-		_, err = vrfPool.AcquireUniqueInteger(vrf)
+		err = acquireVRF(r.ds, vrf)
 		if err != nil {
 			if !metal.IsConflict(err) {
 				if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not acquire vrf: %v", err)) {
@@ -469,11 +463,7 @@ func (r networkResource) allocateNetwork(request *restful.Request, response *res
 }
 
 func createChildNetwork(ds *datastore.RethinkStore, ipamer ipam.IPAMer, nwSpec *metal.Network, parent *metal.Network, childLength int) (*metal.Network, error) {
-	vrfPool, err := ds.GetIntegerPool(datastore.VRFIntegerPool)
-	if err != nil {
-		return nil, err
-	}
-	vrf, err := vrfPool.AcquireRandomUniqueInteger()
+	vrf, err := acquireRandomVRF(ds)
 	if err != nil {
 		return nil, fmt.Errorf("Could not acquire a vrf: %v", err)
 	}
@@ -499,7 +489,7 @@ func createChildNetwork(ds *datastore.RethinkStore, ipamer ipam.IPAMer, nwSpec *
 		Nat:                 parent.Nat,
 		PrivateSuper:        false,
 		Underlay:            false,
-		Vrf:                 vrf,
+		Vrf:                 *vrf,
 		ParentNetworkID:     parent.ID,
 		Labels:              nwSpec.Labels,
 	}
@@ -540,13 +530,7 @@ func (r networkResource) freeNetwork(request *restful.Request, response *restful
 	}
 
 	if nw.Vrf != 0 {
-		vrfPool, err := r.ds.GetIntegerPool(datastore.VRFIntegerPool)
-		if err != nil {
-			if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not release vrf: %v", err)) {
-				return
-			}
-		}
-		err = vrfPool.ReleaseUniqueInteger(nw.Vrf)
+		err = releaseVRF(r.ds, nw.Vrf)
 		if err != nil {
 			if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not release vrf: %v", err)) {
 				return
@@ -689,13 +673,7 @@ func (r networkResource) deleteNetwork(request *restful.Request, response *restf
 	}
 
 	if nw.Vrf != 0 {
-		vrfPool, err := r.ds.GetIntegerPool(datastore.VRFIntegerPool)
-		if err != nil {
-			if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not release vrf: %v", err)) {
-				return
-			}
-		}
-		err = vrfPool.ReleaseUniqueInteger(nw.Vrf)
+		err = releaseVRF(r.ds, nw.Vrf)
 		if err != nil {
 			if checkError(request, response, utils.CurrentFuncName(), fmt.Errorf("could not release vrf: %v", err)) {
 				return

cmd/metal-api/internal/service/v1/machine.go

@@ -51,10 +51,11 @@ type BootInfo struct {
 }
 
 type MachineNetwork struct {
-	NetworkID           string   `json:"networkid" description:"the networkID of the allocated machine in this vrf"`
-	Prefixes            []string `json:"prefixes" description:"the prefixes of this network"`
-	IPs                 []string `json:"ips" description:"the ip addresses of the allocated machine in this vrf"`
-	Vrf                 uint     `json:"vrf" description:"the vrf of the allocated machine"`
+	NetworkID string   `json:"networkid" description:"the networkID of the allocated machine in this vrf"`
+	Prefixes  []string `json:"prefixes" description:"the prefixes of this network"`
+	IPs       []string `json:"ips" description:"the ip addresses of the allocated machine in this vrf"`
+	Vrf       uint     `json:"vrf" description:"the vrf of the allocated machine"`
+	// Attention, uint32 is converted to integer by swagger which is int32 which is to small to hold a asn
 	ASN                 int64    `json:"asn" description:"ASN number for this network in the bgp configuration"`
 	Private             bool     `json:"private" description:"indicates whether this network is the private network of this machine"`
 	Nat                 bool     `json:"nat" description:"if set to true, packets leaving this network get masqueraded behind interface ip"`
@@ -410,7 +411,7 @@ func NewMachineResponse(m *metal.Machine, s *metal.Size, p *metal.Partition, i *
 				NetworkID:           nw.NetworkID,
 				IPs:                 ips,
 				Vrf:                 nw.Vrf,
-				ASN:                 nw.ASN,
+				ASN:                 int64(nw.ASN),
 				Private:             nw.Private,
 				Nat:                 nw.Nat,
 				Underlay:            nw.Underlay,

cmd/metal-api/internal/service/vrf.go

@@ -0,0 +1,36 @@
+package service
+
+import (
+	"fmt"
+
+	"github.com/metal-stack/metal-api/cmd/metal-api/internal/datastore"
+)
+
+// acquireRandomVRF will grab a unique but random vrf out of the vrfintegerpool
+func acquireRandomVRF(ds *datastore.RethinkStore) (*uint, error) {
+	vrfPool, err := ds.GetIntegerPool(datastore.VRFIntegerPool)
+	if err != nil {
+		return nil, fmt.Errorf("could not acquire random vrf: %v", err)
+	}
+	vrf, err := vrfPool.AcquireRandomUniqueInteger()
+	return &vrf, err
+}
+
+// acquireVRF will the given vrf out of the vrfintegerpool if not available a error is thrown
+func acquireVRF(ds *datastore.RethinkStore, vrf uint) error {
+	vrfPool, err := ds.GetIntegerPool(datastore.VRFIntegerPool)
+	if err != nil {
+		return fmt.Errorf("could not acquire vrf:%d %v", vrf, err)
+	}
+	_, err = vrfPool.AcquireUniqueInteger(vrf)
+	return err
+}
+
+// releaseVRF will return the given vrf to the vrfintegerpool for reuse
+func releaseVRF(ds *datastore.RethinkStore, vrf uint) error {
+	vrfPool, err := ds.GetIntegerPool(datastore.VRFIntegerPool)
+	if err != nil {
+		return fmt.Errorf("could not release vrf: %v", err)
+	}
+	return vrfPool.ReleaseUniqueInteger(vrf)
+}