From 74ebd05af28a7e4d44d2df14e8c215643942eaeb Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Wed, 16 Oct 2024 16:20:51 -0500 Subject: [PATCH 1/2] grok support --- go.mod | 2 ++ go.sum | 4 ++++ pkg/observability/grok.go | 41 ++++++++++++++++++++++++++++++++++++++ pkg/observability/types.go | 10 ++++++++++ 4 files changed, 57 insertions(+) create mode 100644 pkg/observability/grok.go diff --git a/go.mod b/go.mod index 3de7ffc..a3c40df 100644 --- a/go.mod +++ b/go.mod @@ -19,11 +19,13 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect github.com/aws/smithy-go v1.22.0 // indirect github.com/beevik/etree v1.4.1 // indirect + github.com/elastic/go-grok v0.3.1 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/in4it/go-devops-platform v0.1.1-0.20241015201149-20da0ccd1259 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect + github.com/magefile/mage v1.15.0 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/russellhaering/gosaml2 v0.9.1 // indirect github.com/russellhaering/goxmldsig v1.4.0 // indirect diff --git a/go.sum b/go.sum index f7f1133..c695a70 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/beevik/etree v1.4.1 h1:PmQJDDYahBGNKDcpdX8uPy1xRCwoCGVUiW669MEirVI= github.com/beevik/etree v1.4.1/go.mod h1:gPNJNaBGVZ9AwsidazFZyygnd+0pAU38N4D+WemwKNs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U= +github.com/elastic/go-grok v0.3.1/go.mod h1:n38ls8ZgOboZRgKcjMY8eFeZFMmcL9n2lP0iHhIDk64= github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= @@ -59,6 +61,8 @@ github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NB github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/pkg/observability/grok.go b/pkg/observability/grok.go new file mode 100644 index 0000000..73eae18 --- /dev/null +++ b/pkg/observability/grok.go @@ -0,0 +1,41 @@ +package observability + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/go-grok" + "github.com/in4it/go-devops-platform/storage" +) + +func compilePatternDefinition(patternDefinition PatternDefinition) (*grok.Grok, error) { + g := grok.New() + g.AddPatterns(patternDefinition.Patterns) + err := g.Compile("%{"+patternDefinition.Name+"}", true) + if err != nil { + return nil, fmt.Errorf("grok compile error: %s", err) + } + return g, nil +} + +func getPatternDefinitions(storage storage.Iface) ([]PatternDefinition, error) { + patternDefinition := []PatternDefinition{} + logConfig, err := getLogConfig(storage) + if err != nil { + return patternDefinition, fmt.Errorf("getLogConfig error: %s", err) + } + return logConfig.PatternDefinitions, nil +} + +func getLogConfig(storage storage.Iface) (LogConfig, error) { + var logConfig LogConfig + contents, err := storage.ReadFile("log-config.json") + if err != nil { + return logConfig, fmt.Errorf("can't read log-config.json: %s", err) + } + err = json.Unmarshal(contents, &logConfig) + if err != nil { + return logConfig, fmt.Errorf("json unmarshal error: %s", err) + } + return logConfig, err +} diff --git a/pkg/observability/types.go b/pkg/observability/types.go index c294c39..1a1a11c 100644 --- a/pkg/observability/types.go +++ b/pkg/observability/types.go @@ -87,3 +87,13 @@ func (kv KeyValueInt) Less(i, j int) bool { func (kv KeyValueInt) Swap(i, j int) { kv[i], kv[j] = kv[j], kv[i] } + +type LogConfig struct { + PatternDefinitions []PatternDefinition `json:"patternDefinitions"` +} + +type PatternDefinition struct { + Name string `json:"name"` + Tags []KeyValue `json:"tags"` + Patterns map[string]string `json:"patterns"` +} From 5bb37669b9010a17537d3845caad9fa7de26b1f5 Mon Sep 17 00:00:00 2001 From: Edward Viaene Date: Wed, 16 Oct 2024 16:38:40 -0500 Subject: [PATCH 2/2] tag checker --- pkg/observability/logs.go | 30 +++++++++++++ pkg/observability/logs_test.go | 79 ++++++++++++++++++++++++++++++++++ pkg/observability/types.go | 2 + 3 files changed, 111 insertions(+) diff --git a/pkg/observability/logs.go b/pkg/observability/logs.go index 772770b..9af086c 100644 --- a/pkg/observability/logs.go +++ b/pkg/observability/logs.go @@ -15,6 +15,11 @@ func (o *Observability) getLogs(fromDate, endDate time.Time, pos int64, maxLogLi Tags: KeyValueInt{}, } + patternDefinitions, err := getPatternDefinitions(o.Storage) + if err != nil { + return logEntryResponse, fmt.Errorf("get pattern definitions error: %s", err) + } + keys := make(map[KeyValue]int) logFiles := []string{} @@ -78,6 +83,13 @@ func (o *Observability) getLogs(fromDate, endDate time.Time, pos int64, maxLogLi } } if !filterMessage { + // check if we need to apply grok filter + for _, patternDefinition := range patternDefinitions { + if logEntryHasTags(patternDefinition.Tags, tags) { // apply grok filter + + } + } + // add log entry logEntry := LogEntry{ Timestamp: timestamp.Format(TIMESTAMP_FORMAT), Data: logline, @@ -114,3 +126,21 @@ func (o *Observability) getLogs(fromDate, endDate time.Time, pos int64, maxLogLi return logEntryResponse, nil } + +func logEntryHasTags(a []KeyValue, b []KeyValue) bool { + tags := make([]bool, len(b)) + for _, a1 := range a { + for k, b1 := range b { + if b1.Key == a1.Key && b1.Value == a1.Value { + tags[k] = true + break + } + } + } + for _, v := range tags { + if !v { + return false + } + } + return len(tags) > 0 +} diff --git a/pkg/observability/logs_test.go b/pkg/observability/logs_test.go index 83e903f..26cf64b 100644 --- a/pkg/observability/logs_test.go +++ b/pkg/observability/logs_test.go @@ -94,3 +94,82 @@ func TestKeyValue(t *testing.T) { t.Fatalf("wrong output: %s", out) } } + +func TestLogEntryHasTags1(t *testing.T) { + a := []KeyValue{ + { + Key: "a", + Value: "b", + }, + } + b := []KeyValue{ + { + Key: "a", + Value: "b", + }, + } + res := logEntryHasTags(a, b) + if !res { + t.Fatalf("res is false") + } +} + +func TestLogEntryHasTags2(t *testing.T) { + a := []KeyValue{ + { + Key: "a", + Value: "b", + }, + } + b := []KeyValue{} + res := logEntryHasTags(a, b) + if res { + t.Fatalf("res is true") + } +} + +func TestLogEntryHasTags3(t *testing.T) { + a := []KeyValue{ + { + Key: "a", + Value: "b", + }, + } + b := []KeyValue{ + { + Key: "a", + Value: "c", + }, + } + res := logEntryHasTags(a, b) + if res { + t.Fatalf("res is true") + } +} + +func TestLogEntryHasTags4(t *testing.T) { + a := []KeyValue{ + { + Key: "a", + Value: "b", + }, + { + Key: "b", + Value: "c", + }, + } + b := []KeyValue{ + { + Key: "b", + Value: "c", + }, + { + Key: "a", + Value: "b", + }, + } + res := logEntryHasTags(a, b) + if !res { + t.Fatalf("res is false") + } +} diff --git a/pkg/observability/types.go b/pkg/observability/types.go index 1a1a11c..c0a3cf8 100644 --- a/pkg/observability/types.go +++ b/pkg/observability/types.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "time" + "github.com/elastic/go-grok" "github.com/in4it/go-devops-platform/storage" ) @@ -27,6 +28,7 @@ type Observability struct { ActiveBufferWriters sync.WaitGroup WriteLock sync.Mutex MaxBufferSize int + GrokPatterns map[string]*grok.Grok } type ConcurrentRWBuffer struct {