Skip to content

Commit

Permalink
Fix for preallocation leftovers on allocation failures (#196)
Browse files Browse the repository at this point in the history
  • Loading branch information
majst01 authored Jun 16, 2021
1 parent a841ff6 commit a61c6ed
Show file tree
Hide file tree
Showing 5 changed files with 863 additions and 131 deletions.
6 changes: 3 additions & 3 deletions cmd/metal-api/internal/service/async-actor.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ type asyncActor struct {
*zap.Logger
ipam.IPAMer
*datastore.RethinkStore
machineReleaser bus.Func
machineNetworkReleaser bus.Func
ipReleaser bus.Func
}

Expand All @@ -28,7 +28,7 @@ func newAsyncActor(l *zap.Logger, ep *bus.Endpoints, ds *datastore.RethinkStore,
RethinkStore: ds,
}
var err error
_, actor.machineReleaser, err = ep.Function("releaseMachineNetworks", actor.releaseMachineNetworks)
_, actor.machineNetworkReleaser, err = ep.Function("releaseMachineNetworks", actor.releaseMachineNetworks)
if err != nil {
return nil, fmt.Errorf("cannot create async bus function for machine releasing: %w", err)
}
Expand All @@ -55,7 +55,7 @@ func (a *asyncActor) freeMachine(pub bus.Publisher, m *metal.Machine) error {
}

// call the releaser async
err = a.machineReleaser(m)
err = a.machineNetworkReleaser(m)
if err != nil {
// log error, but what should we do here? we already called
// deleteVRFSwitches and publishDeleteEvent, so should we return
Expand Down
11 changes: 10 additions & 1 deletion cmd/metal-api/internal/service/machine-service.go
Original file line number Diff line number Diff line change
Expand Up @@ -1047,10 +1047,19 @@ func allocateMachine(logger *zap.SugaredLogger, ds *datastore.RethinkStore, ipam
},
Allocation: alloc,
}
rollbackError := actor.machineReleaser(cleanupMachine)
rollbackError := actor.machineNetworkReleaser(cleanupMachine)
if rollbackError != nil {
logger.Errorw("cannot call async machine cleanup", "error", rollbackError)
}
old := machineCandidate
machineCandidate.Allocation = nil
machineCandidate.Tags = nil
machineCandidate.PreAllocated = false

rollbackError = ds.UpdateMachine(old, machineCandidate)
if rollbackError != nil {
logger.Errorw("cannot update machinecandidate to reset allocation", "error", rollbackError)
}
}
return err
}
Expand Down
34 changes: 17 additions & 17 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,40 +5,40 @@ go 1.16
require (
github.com/Masterminds/semver/v3 v3.1.1
github.com/avast/retry-go v3.0.0+incompatible
github.com/aws/aws-sdk-go v1.37.32
github.com/aws/aws-sdk-go v1.38.61
github.com/dustin/go-humanize v1.0.0
github.com/emicklei/go-restful-openapi/v2 v2.2.1
github.com/emicklei/go-restful/v3 v3.4.0
github.com/emicklei/go-restful-openapi/v2 v2.3.0
github.com/emicklei/go-restful/v3 v3.5.1
github.com/go-logr/zapr v0.4.0
github.com/go-openapi/spec v0.19.15
github.com/go-openapi/spec v0.20.3
github.com/go-stack/stack v1.8.0
github.com/google/go-cmp v0.5.5
github.com/google/go-cmp v0.5.6
github.com/google/uuid v1.2.0
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
github.com/gopherjs/gopherjs v0.0.0-20210614142143-03070f63dd63 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/metal-stack/go-ipam v1.8.4
github.com/metal-stack/masterdata-api v0.8.5
github.com/metal-stack/metal-lib v0.7.2
github.com/metal-stack/go-ipam v1.8.5
github.com/metal-stack/masterdata-api v0.8.7
github.com/metal-stack/metal-lib v0.8.0
github.com/metal-stack/security v0.5.3
github.com/metal-stack/v v1.0.3
github.com/morikuni/aec v1.0.0 // indirect
github.com/nsqio/go-nsq v1.0.8
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.9.0
github.com/prometheus/client_golang v1.11.0
github.com/smartystreets/assertions v1.2.0 // indirect
github.com/spf13/afero v1.4.1 // indirect
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/cobra v1.1.3
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.7.0
github.com/testcontainers/testcontainers-go v0.10.0
go.uber.org/zap v1.16.0
github.com/testcontainers/testcontainers-go v0.11.0
go.uber.org/zap v1.17.0
go4.org v0.0.0-20201209231011-d4a079459e60
golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
google.golang.org/grpc v1.36.0
google.golang.org/protobuf v1.25.0
google.golang.org/grpc v1.38.0
google.golang.org/protobuf v1.26.0
gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1
)
Loading

0 comments on commit a61c6ed

Please sign in to comment.