monitor evenless changes

Author: kendavis2

component/app/model_app.go

@@ -11,7 +11,25 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
-const Undetermined = "undetermined"
+const (
+	AppTypeService       = "service"
+	AppTypeScheduledTask = "scheduled-task"
+	AppTypeLambda        = "lambda"
+	AppTypeS3            = "s3"
+
+	// FailedTaskExpiration determines the minutes an AWS ECS failed tasks should be considered.
+	//
+	// Currently the AWS ECS container restart throttle may wait a maximum of 15 minutes before
+	// attempting a restart. 20 minutes has been chosen as the time to consider failed tasks.
+	// This ensures a service does not appear to be healthy due to lack of attempted restarts
+	// while reducing the time it takes to determine a service is healthy from 1 hour
+	// down to 20 minutes.
+	//
+	// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/service-throttle-logic.html
+	FailedTaskExpiration = 20
+
+	Undetermined = "undetermined"
+)
 
 // Application ...
 type Application struct {
@@ -56,6 +74,19 @@ func (a Application) GetInstances(filter []string) map[string]Instance {
 	return instances
 }
 
+// GetErrorInstances ...
+func (a Application) GetErrorInstances() map[string]Instance {
+	instances := map[string]Instance{}
+
+	for k, i := range a.Instances {
+		if i.CurrentState.Error != nil {
+			instances[k] = i
+		}
+	}
+
+	return instances
+}
+
 // Matches ...
 func (a Application) Matches(filter Filter) bool {
 	for _, t := range filter.Terms {

component/build/registry.go

@@ -24,13 +24,6 @@ type buildView struct {
 	Registry bool `json:"registry"`
 }
 
-const (
-	appTypeService       = "service"
-	appTypeScheduledTask = "scheduled-task"
-	appTypeLambda        = "lambda"
-	appTypeS3            = "s3"
-)
-
 // GetBuilds ...
 func GetBuilds(c echo.Context) error {
 	ctx := c.Get("ctx").(mongo.SessionContext)
@@ -52,7 +45,7 @@ func GetBuilds(c echo.Context) error {
 	builds := map[string]app.Definition{}
 
 	switch apps[0].Type {
-	case appTypeService, appTypeScheduledTask:
+	case app.AppTypeService, app.AppTypeScheduledTask:
 		builds, err = task.ListDefinitions(sourceRegistry.Task)
 		if err != nil {
 			return err
@@ -83,7 +76,7 @@ func GetBuilds(c echo.Context) error {
 			builds[sourceRegistry.FormatVersion()] = sourceRegistry.Task.Definition
 		}
 
-	case appTypeLambda:
+	case app.AppTypeLambda:
 		if len(sourceRegistry.S3RegistryBucket) > 0 {
 			builds, err = s3.ListDefinitions(sourceRegistry)
 			if err != nil {
@@ -95,7 +88,7 @@ func GetBuilds(c echo.Context) error {
 				return err
 			}
 		}
-	case appTypeS3:
+	case app.AppTypeS3:
 		builds, err = s3.ListDefinitions(sourceRegistry)
 		if err != nil {
 			return err

component/integration/aws/service/populate.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"time"
 
 	"github.com/turnerlabs/udeploy/component/app"
 
@@ -78,7 +79,7 @@ func populateInst(i app.Instance, scalableTargets []*applicationautoscaling.Scal
 
 	tasks := append(runningTasks, stoppedTasks...)
 
-	if err := checkError(svcs, stoppedTasks, i); err != nil {
+	if err := checkError(svcs, stoppedTasks, app.FailedTaskExpiration*time.Minute); err != nil {
 		state.SetError(err)
 		state.SetPending()
 	} else if isPending(svcs) {
@@ -108,17 +109,30 @@ func populateInst(i app.Instance, scalableTargets []*applicationautoscaling.Scal
 		return i, state, err
 	}
 
-	i.Links = append(i.Links, app.Link{
-		Generated:   true,
-		Description: "AWS Console Service Logs",
-		Name:        "logs",
-		URL: fmt.Sprintf("https://console.aws.amazon.com/ecs/home?region=%s#/clusters/%s/services/%s/logs",
-			region, i.Cluster, i.Service),
-	})
+	linkName := "logs"
+	if missingLink(linkName, i.Links) {
+		i.Links = append(i.Links, app.Link{
+			Generated:   true,
+			Description: "AWS Console Service Logs",
+			Name:        linkName,
+			URL: fmt.Sprintf("https://console.aws.amazon.com/ecs/home?region=%s#/clusters/%s/services/%s/logs",
+				region, i.Cluster, i.Service),
+		})
+	}
 
 	return i, state, nil
 }
 
+func missingLink(name string, links []app.Link) bool {
+	for _, l := range links {
+		if l.Generated && l.Name == name {
+			return false
+		}
+	}
+
+	return true
+}
+
 func getRegion(arn string) (string, error) {
 	tag := regexp.MustCompile("([a-z]{2}-[a-z]*-[0-9]{1})")
 
@@ -134,14 +148,14 @@ func getRegion(arn string) (string, error) {
 	return "", errors.New("failed to get region")
 }
 
-func checkError(svcs *ecs.Service, tasks []*ecs.Task, inst app.Instance) error {
+func checkError(svcs *ecs.Service, tasks []*ecs.Task, errorExpiration time.Duration) error {
 
 	if *svcs.DesiredCount == 0 {
 		return nil
 	}
 
-	if count, err := getTaskError(tasks); err != nil {
-		return app.InstanceError{Problem: fmt.Sprintf("%d failed task(s) (%s)", count, err)}
+	if _, err := getServiceError(tasks, errorExpiration); err != nil {
+		return app.InstanceError{Problem: err.Error()}
 	}
 
 	return nil
@@ -182,15 +196,18 @@ func getTaskDetails(svc *ecs.ECS, inst app.Instance, tasks []*ecs.Task, status,
 	return getTaskDetails(svc, inst, tasks, status, nextToken)
 }
 
-func getTaskError(tasks []*ecs.Task) (int, error) {
+func getServiceError(tasks []*ecs.Task, expiration time.Duration) (int, error) {
 	var reason error
 	count := 0
 
 	for _, t := range tasks {
 		if t.StopCode != nil && t.StoppedReason != nil {
 			if *t.StopCode != ecs.TaskStopCodeUserInitiated {
-				reason = errors.New(*t.StoppedReason)
-				count++
+
+				if time.Now().Sub(*t.ExecutionStoppedAt) < expiration {
+					reason = errors.New(*t.StoppedReason)
+					count++
+				}
 			}
 		}
 	}
@@ -199,8 +216,6 @@ func getTaskError(tasks []*ecs.Task) (int, error) {
 }
 
 func isPending(svc *ecs.Service) bool {
-
-	//return (len(svc.Deployments) > 1 && *svc.DesiredCount > 0) || *svc.PendingCount > 0 || *svc.DesiredCount > *svc.RunningCount
 	return (len(svc.Deployments) > 1 && *svc.DesiredCount > 0) || *svc.DesiredCount > 0 && *svc.RunningCount == 0
 }
 

component/sync/aws.go

@@ -1,6 +1,9 @@
 package sync
 
 import (
+	"log"
+	"time"
+
 	"github.com/turnerlabs/udeploy/component/action"
 	"github.com/turnerlabs/udeploy/component/app"
 	"github.com/turnerlabs/udeploy/component/cache"
@@ -9,6 +12,49 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
+// AWSPollEventlessChanges monitors and updates the instance state
+// for specific changes that do not trigger AWS events. This polling
+// technique is only used when changes cannot be detected by AWS
+// events. To avoid AWS rate limits, this technique should be used
+// SPARINGLY.
+//
+// Currently AWS does not fire events when errors expire from ECS Task
+// history. Since monitored errors in ECS Task history cause an a
+// pplication to display an error state, the history must be monitored
+// to determine when a service returns to a healthly state.
+func AWSPollEventlessChanges(ctx mongo.SessionContext) error {
+
+	ticker := time.NewTicker(app.FailedTaskExpiration * time.Minute)
+
+	for {
+		select {
+		case <-ticker.C:
+			for _, a := range cache.Apps.GetAll() {
+				if a.Type != app.AppTypeService {
+					continue
+				}
+
+				targeted := a.GetErrorInstances()
+				if len(targeted) == 0 {
+					continue
+				}
+
+				log.Printf("Updating App: %s\n", a.Name)
+
+				supplemented, err := supplement.Instances(ctx, a.Type, targeted, false)
+				if err != nil {
+					log.Printf("failed to update %s state (%s)\n", a.Name, err)
+					continue
+				}
+
+				cache.Apps.UpdateInstances(a.Name, supplemented)
+
+				time.Sleep(time.Second)
+			}
+		}
+	}
+}
+
 // AWSWatchEvents ...
 func AWSWatchEvents(ctx mongo.SessionContext) error {
 	return sqs.MonitorChanges(ctx, handleChange)

monitor.go

@@ -46,6 +46,15 @@ func monitorChanges(ctx context.Context, sess mongo.Session) {
 		}
 	}()
 
+	go func() {
+		if err := mongo.WithSession(ctx, sess, func(sctx mongo.SessionContext) error {
+			log.Fatal(sync.AWSPollEventlessChanges(sctx))
+			return nil
+		}); err != nil {
+			log.Fatal(err)
+		}
+	}()
+
 	//--------------------------------------------------
 	//- Watch for cloudwatch alarms
 	//--------------------------------------------------