From da712cc16fa3dfa0930279ab70a93e6fd600f9b8 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 18 Sep 2024 11:10:00 +0530 Subject: [PATCH 1/3] fix: enrich attributes regardless if it is materialized --- pkg/query-service/app/logs/v3/enrich_query.go | 19 ++--- .../app/logs/v3/enrich_query_test.go | 76 +++++++++++++++++-- 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/pkg/query-service/app/logs/v3/enrich_query.go b/pkg/query-service/app/logs/v3/enrich_query.go index b8ed0ff801..e461a9a8c4 100644 --- a/pkg/query-service/app/logs/v3/enrich_query.go +++ b/pkg/query-service/app/logs/v3/enrich_query.go @@ -61,26 +61,23 @@ func EnrichmentRequired(params *v3.QueryRangeParamsV3) bool { return false } +// if the field is timestamp/id/value we don't need to enrich +// if the field is static we don't need to enrich +// for all others we need to enrich +// an attribute/resource can be materialized/dematerialized +// but the query should work regardless and shouldn't fail func isEnriched(field v3.AttributeKey) bool { // if it is timestamp/id dont check if field.Key == "timestamp" || field.Key == "id" || field.Key == constants.SigNozOrderByValue { return true } - if field.IsColumn { + // don't need to enrich the static fields as they will be always used a column + if _, ok := constants.StaticFieldsLogsV3[field.Key]; ok && field.IsColumn { return true } - if field.Type == v3.AttributeKeyTypeUnspecified || field.DataType == v3.AttributeKeyDataTypeUnspecified { - return false - } - - // try to enrich all attributes which doesn't have isColumn = true - if !field.IsColumn { - return false - } - - return true + return false } func Enrich(params *v3.QueryRangeParamsV3, fields map[string]v3.AttributeKey) { diff --git a/pkg/query-service/app/logs/v3/enrich_query_test.go b/pkg/query-service/app/logs/v3/enrich_query_test.go index 32d714925c..8623e053db 100644 --- a/pkg/query-service/app/logs/v3/enrich_query_test.go +++ b/pkg/query-service/app/logs/v3/enrich_query_test.go @@ -51,7 +51,7 @@ var testEnrichmentRequiredData = []struct { EnrichmentRequired: true, }, { - Name: "filter enrichment not required", + Name: "filter enrichment required", Params: v3.QueryRangeParamsV3{ CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ @@ -87,7 +87,7 @@ var testEnrichmentRequiredData = []struct { EnrichmentRequired: true, }, { - Name: "filter enrichment not required required json", + Name: "filter enrichment required required json", Params: v3.QueryRangeParamsV3{ CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ @@ -105,7 +105,7 @@ var testEnrichmentRequiredData = []struct { EnrichmentRequired: true, }, { - Name: "groupBy enrichment not required", + Name: "groupBy enrichment required", Params: v3.QueryRangeParamsV3{ CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ @@ -194,8 +194,9 @@ var testEnrichmentRequiredData = []struct { QueryName: "test", Expression: "test", DataSource: v3.DataSourceLogs, - GroupBy: []v3.AttributeKey{{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, - OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}}, + // here we have to fallback to trace_id attribute instead of column + GroupBy: []v3.AttributeKey{{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}}, }, }, }, @@ -393,6 +394,71 @@ var testEnrichParamsData = []struct { }, }, }, + { + Name: "Enrich if an attribute/resource attribute is materialized/dematerialized", + Params: v3.QueryRangeParamsV3{ + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "test": { + QueryName: "test", + Expression: "test", + DataSource: v3.DataSourceLogs, + AggregateAttribute: v3.AttributeKey{ + Key: "mat_resource", + Type: v3.AttributeKeyTypeResource, + DataType: v3.AttributeKeyDataTypeInt64, + IsColumn: true, + }, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "mat_attr", Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "test", Operator: "="}, + {Key: v3.AttributeKey{Key: "normal_attr", DataType: v3.AttributeKeyDataTypeString, IsColumn: false}, Value: "test1", Operator: "="}, + }}, + }, + }, + }, + }, + Fields: map[string]v3.AttributeKey{ + "mat_resource": { + Key: "mat_resource", + Type: v3.AttributeKeyTypeResource, + DataType: v3.AttributeKeyDataTypeInt64, + IsColumn: false, + }, + "mat_attr": { + Key: "mat_attr", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + IsColumn: false, + }, + "normal_attr": { + Key: "normal_attr", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + }, + Result: v3.QueryRangeParamsV3{ + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "test": { + QueryName: "test", + Expression: "test", + DataSource: v3.DataSourceLogs, + AggregateAttribute: v3.AttributeKey{ + Key: "mat_resource", + Type: v3.AttributeKeyTypeResource, + DataType: v3.AttributeKeyDataTypeInt64, + IsColumn: false, + }, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "mat_attr", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, IsColumn: false}, Value: "test", Operator: "="}, + {Key: v3.AttributeKey{Key: "normal_attr", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Value: "test1", Operator: "="}, + }}, + }, + }, + }, + }, + }, } func TestEnrichParams(t *testing.T) { From d8271ad2baa31b7478473a30d442b1131cc8da4f Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 18 Sep 2024 14:57:03 +0530 Subject: [PATCH 2/3] feat: take care of same key name with different type and datatype --- pkg/query-service/app/logs/v3/enrich_query.go | 41 +++++++++-- .../app/logs/v3/enrich_query_test.go | 71 ++++++++++++++++--- pkg/query-service/model/logs.go | 7 +- pkg/query-service/model/v3/v3.go | 8 +++ 4 files changed, 108 insertions(+), 19 deletions(-) diff --git a/pkg/query-service/app/logs/v3/enrich_query.go b/pkg/query-service/app/logs/v3/enrich_query.go index e461a9a8c4..0d9deca9c2 100644 --- a/pkg/query-service/app/logs/v3/enrich_query.go +++ b/pkg/query-service/app/logs/v3/enrich_query.go @@ -130,6 +130,37 @@ func EnrichLogsQuery(query *v3.BuilderQuery, fields map[string]v3.AttributeKey) return nil } +// This tries to see all possible fields that it can fall back to if some meta is missing +// check Test_generateEnrichmentKeys for example +func generateEnrichmentKeys(field v3.AttributeKey) []string { + names := []string{} + if field.Type != v3.AttributeKeyTypeUnspecified && field.DataType != v3.AttributeKeyDataTypeUnspecified { + names = append(names, field.Key+"##"+field.Type.String()+"##"+field.DataType.String()) + return names + } + + types := []string{} + dTypes := []string{} + if field.Type != v3.AttributeKeyTypeUnspecified { + types = append(types, field.Type.String()) + } else { + types = append(types, v3.AttributeKeyTypeTag.String(), v3.AttributeKeyTypeResource.String()) + } + if field.DataType != v3.AttributeKeyDataTypeUnspecified { + dTypes = append(dTypes, field.DataType.String()) + } else { + dTypes = append(dTypes, v3.AttributeKeyDataTypeFloat64.String(), v3.AttributeKeyDataTypeInt64.String(), v3.AttributeKeyDataTypeString.String(), v3.AttributeKeyDataTypeBool.String()) + } + + for _, t := range types { + for _, d := range dTypes { + names = append(names, field.Key+"##"+t+"##"+d) + } + } + + return names +} + func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.AttributeKey) v3.AttributeKey { if isEnriched(field) { return field @@ -141,13 +172,9 @@ func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.Attribu } // check if the field is present in the fields map - if existingField, ok := fields[field.Key]; ok { - // don't update if type is not the same - if (field.Type == "" && field.DataType == "") || - (field.Type == existingField.Type && field.DataType == existingField.DataType) || - (field.Type == "" && field.DataType == existingField.DataType) || - (field.DataType == "" && field.Type == existingField.Type) { - return existingField + for _, key := range generateEnrichmentKeys(field) { + if val, ok := fields[key]; ok { + return val } } diff --git a/pkg/query-service/app/logs/v3/enrich_query_test.go b/pkg/query-service/app/logs/v3/enrich_query_test.go index 8623e053db..26ac5a5061 100644 --- a/pkg/query-service/app/logs/v3/enrich_query_test.go +++ b/pkg/query-service/app/logs/v3/enrich_query_test.go @@ -1,6 +1,7 @@ package v3 import ( + "reflect" "testing" . "github.com/smartystreets/goconvey/convey" @@ -242,17 +243,17 @@ var testEnrichParamsData = []struct { }, }, Fields: map[string]v3.AttributeKey{ - "test": { + "test##tag##int64": { Key: "test", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeInt64, }, - "user_name": { + "user_name##tag##string": { Key: "user_name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, }, - "response_time": { + "response_time##tag##int64": { Key: "response_time", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeInt64, @@ -303,18 +304,18 @@ var testEnrichParamsData = []struct { }, }, Fields: map[string]v3.AttributeKey{ - "method.name": { + "method.name##tag##string": { Key: "method.name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "service.name": { + "service.name##tag##string": { Key: "service.name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, }, - "host.name": { + "host.name##tag##string": { Key: "host.name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, @@ -366,7 +367,7 @@ var testEnrichParamsData = []struct { }, }, Fields: map[string]v3.AttributeKey{ - "test": { + "test##tag##string": { Key: "test", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, @@ -418,19 +419,19 @@ var testEnrichParamsData = []struct { }, }, Fields: map[string]v3.AttributeKey{ - "mat_resource": { + "mat_resource##resource##int64": { Key: "mat_resource", Type: v3.AttributeKeyTypeResource, DataType: v3.AttributeKeyDataTypeInt64, IsColumn: false, }, - "mat_attr": { + "mat_attr##tag##string": { Key: "mat_attr", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, IsColumn: false, }, - "normal_attr": { + "normal_attr##tag##string": { Key: "normal_attr", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, @@ -742,3 +743,53 @@ func TestParseStrValue(t *testing.T) { }) } } + +func Test_generateEnrichmentKeys(t *testing.T) { + type args struct { + field v3.AttributeKey + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "all are present", + args: args{ + field: v3.AttributeKey{ + Key: "data", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + want: []string{"data##tag##string"}, + }, + { + name: "type present", + args: args{ + field: v3.AttributeKey{ + Key: "data", + Type: v3.AttributeKeyTypeTag, + }, + }, + want: []string{"data##tag##float64", "data##tag##int64", "data##tag##string", "data##tag##bool"}, + }, + { + name: "dataType present", + args: args{ + field: v3.AttributeKey{ + Key: "data", + DataType: v3.AttributeKeyDataTypeString, + }, + }, + want: []string{"data##tag##string", "data##resource##string"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := generateEnrichmentKeys(tt.args.field); !reflect.DeepEqual(got, tt.want) { + t.Errorf("generateEnrichmentKeys() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/query-service/model/logs.go b/pkg/query-service/model/logs.go index 0319581516..cc46e63922 100644 --- a/pkg/query-service/model/logs.go +++ b/pkg/query-service/model/logs.go @@ -49,7 +49,8 @@ func GetLogFieldsV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3 if pass { continue } - data[selectedField.Name] = v3.AttributeKey{ + name := selectedField.Name + "##" + fieldType.String() + "##" + selectedField.DataType + data[name] = v3.AttributeKey{ Key: selectedField.Name, Type: fieldType, DataType: v3.AttributeKeyDataType(strings.ToLower(selectedField.DataType)), @@ -61,12 +62,14 @@ func GetLogFieldsV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3 if pass { continue } - data[interestingField.Name] = v3.AttributeKey{ + name := interestingField.Name + "##" + fieldType.String() + "##" + interestingField.DataType + data[name] = v3.AttributeKey{ Key: interestingField.Name, Type: fieldType, DataType: v3.AttributeKeyDataType(strings.ToLower(interestingField.DataType)), IsColumn: false, } + } break } diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 2d99118533..8d88d6c5d6 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -287,6 +287,10 @@ func (q AttributeKeyDataType) Validate() error { } } +func (q AttributeKeyDataType) String() string { + return string(q) +} + // FilterAttributeValueRequest is a request to fetch possible attribute values // for a selected aggregate operator, aggregate attribute, filter attribute key // and search text. @@ -317,6 +321,10 @@ const ( AttributeKeyTypeResource AttributeKeyType = "resource" ) +func (t AttributeKeyType) String() string { + return string(t) +} + type AttributeKey struct { Key string `json:"key"` DataType AttributeKeyDataType `json:"dataType"` From 142c60f5e6db3aa7fb6908506f16285edf7c9323 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 19 Sep 2024 16:08:43 +0530 Subject: [PATCH 3/3] fix: makeLinks updated with new logic for logs --- pkg/query-service/app/http_handler.go | 4 +- pkg/query-service/app/logs/v3/enrich_query.go | 34 +---------- .../app/logs/v3/enrich_query_test.go | 51 ----------------- pkg/query-service/contextlinks/links.go | 20 ++++++- pkg/query-service/model/logs.go | 4 +- pkg/query-service/rules/threshold_rule.go | 4 +- pkg/query-service/utils/logs.go | 33 +++++++++++ pkg/query-service/utils/logs_test.go | 57 ++++++++++++++++++- 8 files changed, 114 insertions(+), 93 deletions(-) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 219181dc7f..39bc0068ce 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -838,7 +838,7 @@ func (aH *APIHandler) getRuleStateHistory(w http.ResponseWriter, r *http.Request continue } filterItems, groupBy, keys := aH.metaForLinks(r.Context(), rule) - newFilters := contextlinks.PrepareFilters(lbls, filterItems, groupBy, keys) + newFilters := contextlinks.PrepareFilters(rule.AlertType == rules.AlertTypeLogs, lbls, filterItems, groupBy, keys) end := time.Unix(res.Items[idx].UnixMilli/1000, 0) // why are we subtracting 3 minutes? // the query range is calculated based on the rule's evalWindow and evalDelay @@ -880,7 +880,7 @@ func (aH *APIHandler) getRuleStateHistoryTopContributors(w http.ResponseWriter, continue } filterItems, groupBy, keys := aH.metaForLinks(r.Context(), rule) - newFilters := contextlinks.PrepareFilters(lbls, filterItems, groupBy, keys) + newFilters := contextlinks.PrepareFilters(rule.AlertType == rules.AlertTypeLogs, lbls, filterItems, groupBy, keys) end := time.Unix(params.End/1000, 0) start := time.Unix(params.Start/1000, 0) if rule.AlertType == rules.AlertTypeLogs { diff --git a/pkg/query-service/app/logs/v3/enrich_query.go b/pkg/query-service/app/logs/v3/enrich_query.go index 0d9deca9c2..49ff422fd5 100644 --- a/pkg/query-service/app/logs/v3/enrich_query.go +++ b/pkg/query-service/app/logs/v3/enrich_query.go @@ -7,6 +7,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" ) func EnrichmentRequired(params *v3.QueryRangeParamsV3) bool { @@ -130,37 +131,6 @@ func EnrichLogsQuery(query *v3.BuilderQuery, fields map[string]v3.AttributeKey) return nil } -// This tries to see all possible fields that it can fall back to if some meta is missing -// check Test_generateEnrichmentKeys for example -func generateEnrichmentKeys(field v3.AttributeKey) []string { - names := []string{} - if field.Type != v3.AttributeKeyTypeUnspecified && field.DataType != v3.AttributeKeyDataTypeUnspecified { - names = append(names, field.Key+"##"+field.Type.String()+"##"+field.DataType.String()) - return names - } - - types := []string{} - dTypes := []string{} - if field.Type != v3.AttributeKeyTypeUnspecified { - types = append(types, field.Type.String()) - } else { - types = append(types, v3.AttributeKeyTypeTag.String(), v3.AttributeKeyTypeResource.String()) - } - if field.DataType != v3.AttributeKeyDataTypeUnspecified { - dTypes = append(dTypes, field.DataType.String()) - } else { - dTypes = append(dTypes, v3.AttributeKeyDataTypeFloat64.String(), v3.AttributeKeyDataTypeInt64.String(), v3.AttributeKeyDataTypeString.String(), v3.AttributeKeyDataTypeBool.String()) - } - - for _, t := range types { - for _, d := range dTypes { - names = append(names, field.Key+"##"+t+"##"+d) - } - } - - return names -} - func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.AttributeKey) v3.AttributeKey { if isEnriched(field) { return field @@ -172,7 +142,7 @@ func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.Attribu } // check if the field is present in the fields map - for _, key := range generateEnrichmentKeys(field) { + for _, key := range utils.GenerateLogEnrichmentKeys(field) { if val, ok := fields[key]; ok { return val } diff --git a/pkg/query-service/app/logs/v3/enrich_query_test.go b/pkg/query-service/app/logs/v3/enrich_query_test.go index 26ac5a5061..20b5d90487 100644 --- a/pkg/query-service/app/logs/v3/enrich_query_test.go +++ b/pkg/query-service/app/logs/v3/enrich_query_test.go @@ -1,7 +1,6 @@ package v3 import ( - "reflect" "testing" . "github.com/smartystreets/goconvey/convey" @@ -743,53 +742,3 @@ func TestParseStrValue(t *testing.T) { }) } } - -func Test_generateEnrichmentKeys(t *testing.T) { - type args struct { - field v3.AttributeKey - } - tests := []struct { - name string - args args - want []string - }{ - { - name: "all are present", - args: args{ - field: v3.AttributeKey{ - Key: "data", - DataType: v3.AttributeKeyDataTypeString, - Type: v3.AttributeKeyTypeTag, - }, - }, - want: []string{"data##tag##string"}, - }, - { - name: "type present", - args: args{ - field: v3.AttributeKey{ - Key: "data", - Type: v3.AttributeKeyTypeTag, - }, - }, - want: []string{"data##tag##float64", "data##tag##int64", "data##tag##string", "data##tag##bool"}, - }, - { - name: "dataType present", - args: args{ - field: v3.AttributeKey{ - Key: "data", - DataType: v3.AttributeKeyDataTypeString, - }, - }, - want: []string{"data##tag##string", "data##resource##string"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := generateEnrichmentKeys(tt.args.field); !reflect.DeepEqual(got, tt.want) { - t.Errorf("generateEnrichmentKeys() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/query-service/contextlinks/links.go b/pkg/query-service/contextlinks/links.go index d0d8400e74..8c3b3180ef 100644 --- a/pkg/query-service/contextlinks/links.go +++ b/pkg/query-service/contextlinks/links.go @@ -8,6 +8,7 @@ import ( tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" ) func PrepareLinksToTraces(start, end time.Time, filterItems []v3.FilterItem) string { @@ -146,7 +147,7 @@ func PrepareLinksToLogs(start, end time.Time, filterItems []v3.FilterItem) strin // by clause, in which case we replace it with the actual value for the notification // i.e Severity text = WARN // If the Severity text is not part of the group by clause, then we add it as it is -func PrepareFilters(labels map[string]string, whereClauseItems []v3.FilterItem, groupByItems []v3.AttributeKey, keys map[string]v3.AttributeKey) []v3.FilterItem { +func PrepareFilters(alertTypeLogs bool, labels map[string]string, whereClauseItems []v3.FilterItem, groupByItems []v3.AttributeKey, keys map[string]v3.AttributeKey) []v3.FilterItem { var filterItems []v3.FilterItem added := make(map[string]struct{}) @@ -178,8 +179,21 @@ func PrepareFilters(labels map[string]string, whereClauseItems []v3.FilterItem, for key, value := range labels { if _, ok := added[key]; !ok { // start by taking the attribute key from the keys map, if not present, create a new one - attributeKey, ok := keys[key] - if !ok { + var attributeKey v3.AttributeKey + var attrFound bool + if alertTypeLogs { + for _, tKey := range utils.GenerateLogEnrichmentKeys(v3.AttributeKey{Key: key}) { + if val, ok := keys[tKey]; ok { + attributeKey = val + attrFound = true + break + } + } + } else { + attributeKey, attrFound = keys[key] + } + + if !attrFound { attributeKey = v3.AttributeKey{Key: key} } diff --git a/pkg/query-service/model/logs.go b/pkg/query-service/model/logs.go index cc46e63922..c6a9276008 100644 --- a/pkg/query-service/model/logs.go +++ b/pkg/query-service/model/logs.go @@ -49,7 +49,7 @@ func GetLogFieldsV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3 if pass { continue } - name := selectedField.Name + "##" + fieldType.String() + "##" + selectedField.DataType + name := selectedField.Name + "##" + fieldType.String() + "##" + strings.ToLower(selectedField.DataType) data[name] = v3.AttributeKey{ Key: selectedField.Name, Type: fieldType, @@ -62,7 +62,7 @@ func GetLogFieldsV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3 if pass { continue } - name := interestingField.Name + "##" + fieldType.String() + "##" + interestingField.DataType + name := interestingField.Name + "##" + fieldType.String() + "##" + strings.ToLower(interestingField.DataType) data[name] = v3.AttributeKey{ Key: interestingField.Name, Type: fieldType, diff --git a/pkg/query-service/rules/threshold_rule.go b/pkg/query-service/rules/threshold_rule.go index 0f768314cf..5b18f5df43 100644 --- a/pkg/query-service/rules/threshold_rule.go +++ b/pkg/query-service/rules/threshold_rule.go @@ -264,7 +264,7 @@ func (r *ThresholdRule) prepareLinksToLogs(ts time.Time, lbls labels.Labels) str queryFilter = q.Filters.Items } - filterItems := contextlinks.PrepareFilters(lbls.Map(), queryFilter, q.GroupBy, r.logsKeys) + filterItems := contextlinks.PrepareFilters(true, lbls.Map(), queryFilter, q.GroupBy, r.logsKeys) return contextlinks.PrepareLinksToLogs(start, end, filterItems) } @@ -298,7 +298,7 @@ func (r *ThresholdRule) prepareLinksToTraces(ts time.Time, lbls labels.Labels) s queryFilter = q.Filters.Items } - filterItems := contextlinks.PrepareFilters(lbls.Map(), queryFilter, q.GroupBy, r.spansKeys) + filterItems := contextlinks.PrepareFilters(false, lbls.Map(), queryFilter, q.GroupBy, r.spansKeys) return contextlinks.PrepareLinksToTraces(start, end, filterItems) } diff --git a/pkg/query-service/utils/logs.go b/pkg/query-service/utils/logs.go index 2f536ef857..8efa026b52 100644 --- a/pkg/query-service/utils/logs.go +++ b/pkg/query-service/utils/logs.go @@ -1,5 +1,7 @@ package utils +import v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + const HOUR_NANO = int64(3600000000000) type LogsListTsRange struct { @@ -36,3 +38,34 @@ func GetLogsListTsRanges(start, end int64) []LogsListTsRange { } return result } + +// This tries to see all possible fields that it can fall back to if some meta is missing +// check Test_GenerateLogEnrichmentKeys for example +func GenerateLogEnrichmentKeys(field v3.AttributeKey) []string { + names := []string{} + if field.Type != v3.AttributeKeyTypeUnspecified && field.DataType != v3.AttributeKeyDataTypeUnspecified { + names = append(names, field.Key+"##"+field.Type.String()+"##"+field.DataType.String()) + return names + } + + types := []v3.AttributeKeyType{} + dTypes := []v3.AttributeKeyDataType{} + if field.Type != v3.AttributeKeyTypeUnspecified { + types = append(types, field.Type) + } else { + types = append(types, v3.AttributeKeyTypeTag, v3.AttributeKeyTypeResource) + } + if field.DataType != v3.AttributeKeyDataTypeUnspecified { + dTypes = append(dTypes, field.DataType) + } else { + dTypes = append(dTypes, v3.AttributeKeyDataTypeFloat64, v3.AttributeKeyDataTypeInt64, v3.AttributeKeyDataTypeString, v3.AttributeKeyDataTypeBool) + } + + for _, t := range types { + for _, d := range dTypes { + names = append(names, field.Key+"##"+t.String()+"##"+d.String()) + } + } + + return names +} diff --git a/pkg/query-service/utils/logs_test.go b/pkg/query-service/utils/logs_test.go index 939fa5fa1b..e1efd813d1 100644 --- a/pkg/query-service/utils/logs_test.go +++ b/pkg/query-service/utils/logs_test.go @@ -1,6 +1,11 @@ package utils -import "testing" +import ( + "reflect" + "testing" + + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) func TestLogsListTsRange(t *testing.T) { startEndData := []struct { @@ -47,3 +52,53 @@ func TestLogsListTsRange(t *testing.T) { } } } + +func Test_GenerateLogEnrichmentKeys(t *testing.T) { + type args struct { + field v3.AttributeKey + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "all are present", + args: args{ + field: v3.AttributeKey{ + Key: "data", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + want: []string{"data##tag##string"}, + }, + { + name: "type present", + args: args{ + field: v3.AttributeKey{ + Key: "data", + Type: v3.AttributeKeyTypeTag, + }, + }, + want: []string{"data##tag##float64", "data##tag##int64", "data##tag##string", "data##tag##bool"}, + }, + { + name: "dataType present", + args: args{ + field: v3.AttributeKey{ + Key: "data", + DataType: v3.AttributeKeyDataTypeString, + }, + }, + want: []string{"data##tag##string", "data##resource##string"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GenerateLogEnrichmentKeys(tt.args.field); !reflect.DeepEqual(got, tt.want) { + t.Errorf("generateLogEnrichmentKeys() = %v, want %v", got, tt.want) + } + }) + } +}