diff --git a/go.mod b/go.mod index c7cafd89cc..9d61916d42 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/sethvargo/go-password v0.2.0 github.com/smartystreets/goconvey v1.8.1 github.com/soheilhy/cmux v0.1.5 - github.com/srikanthccv/ClickHouse-go-mock v0.8.0 + github.com/srikanthccv/ClickHouse-go-mock v0.9.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.103.0 go.opentelemetry.io/collector/confmap v0.103.0 diff --git a/go.sum b/go.sum index a98137d6db..a442200b0e 100644 --- a/go.sum +++ b/go.sum @@ -716,6 +716,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/srikanthccv/ClickHouse-go-mock v0.8.0 h1:DeeM8XLbTFl6sjYPPwazPEXx7kmRV8TgPFVkt1SqT0Y= github.com/srikanthccv/ClickHouse-go-mock v0.8.0/go.mod h1:pgJm+apjvi7FHxEdgw1Bt4MRbUYpVxyhKQ/59Wkig24= +github.com/srikanthccv/ClickHouse-go-mock v0.9.0 h1:XKr1Tb7GL1HlifKH874QGR3R6l0e6takXasROUiZawU= +github.com/srikanthccv/ClickHouse-go-mock v0.9.0/go.mod h1:pgJm+apjvi7FHxEdgw1Bt4MRbUYpVxyhKQ/59Wkig24= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 5d8f7ece82..bac50ca157 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -4414,7 +4414,7 @@ func (r *ClickHouseReader) GetQBFilterSuggestionsForLogs( ctx, &v3.FilterAttributeKeyRequest{ SearchText: req.SearchText, DataSource: v3.DataSourceLogs, - Limit: req.Limit, + Limit: int(req.AttributesLimit), }) if err != nil { return nil, model.InternalError(fmt.Errorf("couldn't get attribute keys: %w", err)) @@ -4458,53 +4458,61 @@ func (r *ClickHouseReader) GetQBFilterSuggestionsForLogs( } } - // Suggest example query for top suggested attribute using existing - // autocomplete logic for recommending attrib values - // - // Example queries for multiple top attributes using a batch version of - // GetLogAttributeValues is expected to come in a follow up change - if len(suggestions.AttributeKeys) > 0 { - topAttrib := suggestions.AttributeKeys[0] - - resp, err := r.GetLogAttributeValues(ctx, &v3.FilterAttributeValueRequest{ - DataSource: v3.DataSourceLogs, - FilterAttributeKey: topAttrib.Key, - FilterAttributeKeyDataType: topAttrib.DataType, - TagType: v3.TagType(topAttrib.Type), - Limit: 1, - }) + // Suggest example queries for top suggested log attributes and resource attributes + exampleAttribs := []v3.AttributeKey{} + for _, attrib := range suggestions.AttributeKeys { + isAttributeOrResource := slices.Contains([]v3.AttributeKeyType{ + v3.AttributeKeyTypeResource, v3.AttributeKeyTypeTag, + }, attrib.Type) + + isNumOrStringType := slices.Contains([]v3.AttributeKeyDataType{ + v3.AttributeKeyDataTypeInt64, v3.AttributeKeyDataTypeFloat64, v3.AttributeKeyDataTypeString, + }, attrib.DataType) + + if isAttributeOrResource && isNumOrStringType { + exampleAttribs = append(exampleAttribs, attrib) + } + + if len(exampleAttribs) >= int(req.ExamplesLimit) { + break + } + } + if len(exampleAttribs) > 0 { + exampleAttribValues, err := r.getValuesForLogAttributes( + ctx, exampleAttribs, req.ExamplesLimit, + ) if err != nil { // Do not fail the entire request if only example query generation fails zap.L().Error("could not find attribute values for creating example query", zap.Error(err)) - } else { - addExampleQuerySuggestion := func(value any) { - exampleQuery := newExampleQuery() - - exampleQuery.Items = append(exampleQuery.Items, v3.FilterItem{ - Key: topAttrib, - Operator: "=", - Value: value, - }) - - suggestions.ExampleQueries = append( - suggestions.ExampleQueries, exampleQuery, - ) - } - if len(resp.StringAttributeValues) > 0 { - addExampleQuerySuggestion(resp.StringAttributeValues[0]) - } else if len(resp.NumberAttributeValues) > 0 { - addExampleQuerySuggestion(resp.NumberAttributeValues[0]) - } else if len(resp.BoolAttributeValues) > 0 { - addExampleQuerySuggestion(resp.BoolAttributeValues[0]) + // add example queries for as many attributes as possible. + // suggest 1st value for 1st attrib, followed by 1st value for second attrib and so on + // and if there is still room, suggest 2nd value for 1st attrib, 2nd value for 2nd attrib and so on + for valueIdx := 0; valueIdx < int(req.ExamplesLimit); valueIdx++ { + for attrIdx, attr := range exampleAttribs { + needMoreExamples := len(suggestions.ExampleQueries) < int(req.ExamplesLimit) + + if needMoreExamples && valueIdx < len(exampleAttribValues[attrIdx]) { + exampleQuery := newExampleQuery() + exampleQuery.Items = append(exampleQuery.Items, v3.FilterItem{ + Key: attr, + Operator: "=", + Value: exampleAttribValues[attrIdx][valueIdx], + }) + + suggestions.ExampleQueries = append( + suggestions.ExampleQueries, exampleQuery, + ) + } + } } } } // Suggest static example queries for standard log attributes if needed. - if len(suggestions.ExampleQueries) < req.Limit { + if len(suggestions.ExampleQueries) < int(req.ExamplesLimit) { exampleQuery := newExampleQuery() exampleQuery.Items = append(exampleQuery.Items, v3.FilterItem{ Key: v3.AttributeKey{ @@ -4522,6 +4530,108 @@ func (r *ClickHouseReader) GetQBFilterSuggestionsForLogs( return &suggestions, nil } +// Get up to `limit` values seen for each attribute in `attributes` +// Returns a slice of slices where the ith slice has values for ith entry in `attributes` +func (r *ClickHouseReader) getValuesForLogAttributes( + ctx context.Context, attributes []v3.AttributeKey, limit uint64, +) ([][]any, *model.ApiError) { + // query top `limit` distinct values seen for `tagKey`s of interest + // ordered by timestamp when the value was seen + query := fmt.Sprintf( + ` + select tagKey, stringTagValue, int64TagValue, float64TagValue + from ( + select + tagKey, + stringTagValue, + int64TagValue, + float64TagValue, + row_number() over (partition by tagKey order by ts desc) as rank + from ( + select + tagKey, + stringTagValue, + int64TagValue, + float64TagValue, + max(timestamp) as ts + from %s.%s + where tagKey in $1 + group by (tagKey, stringTagValue, int64TagValue, float64TagValue) + ) + ) + where rank <= %d + `, + r.logsDB, r.logsTagAttributeTable, limit, + ) + + attribNames := []string{} + for _, attrib := range attributes { + attribNames = append(attribNames, attrib.Key) + } + + rows, err := r.db.Query(ctx, query, attribNames) + if err != nil { + zap.L().Error("couldn't query attrib values for suggestions", zap.Error(err)) + return nil, model.InternalError(fmt.Errorf( + "couldn't query attrib values for suggestions: %w", err, + )) + } + defer rows.Close() + + result := make([][]any, len(attributes)) + + // Helper for getting hold of the result slice to append to for each scanned row + resultIdxForAttrib := func(key string, dataType v3.AttributeKeyDataType) int { + return slices.IndexFunc(attributes, func(attrib v3.AttributeKey) bool { + return attrib.Key == key && attrib.DataType == dataType + }) + } + + // Scan rows and append to result + for rows.Next() { + var tagKey string + var stringValue string + var float64Value sql.NullFloat64 + var int64Value sql.NullInt64 + + err := rows.Scan( + &tagKey, &stringValue, &int64Value, &float64Value, + ) + if err != nil { + return nil, model.InternalError(fmt.Errorf( + "couldn't scan attrib value rows: %w", err, + )) + } + + if len(stringValue) > 0 { + attrResultIdx := resultIdxForAttrib(tagKey, v3.AttributeKeyDataTypeString) + if attrResultIdx >= 0 { + result[attrResultIdx] = append(result[attrResultIdx], stringValue) + } + + } else if int64Value.Valid { + attrResultIdx := resultIdxForAttrib(tagKey, v3.AttributeKeyDataTypeInt64) + if attrResultIdx >= 0 { + result[attrResultIdx] = append(result[attrResultIdx], int64Value.Int64) + } + + } else if float64Value.Valid { + attrResultIdx := resultIdxForAttrib(tagKey, v3.AttributeKeyDataTypeFloat64) + if attrResultIdx >= 0 { + result[attrResultIdx] = append(result[attrResultIdx], float64Value.Float64) + } + } + } + + if err := rows.Err(); err != nil { + return nil, model.InternalError(fmt.Errorf( + "couldn't scan attrib value rows: %w", err, + )) + } + + return result, nil +} + func readRow(vars []interface{}, columnNames []string, countOfNumberCols int) ([]string, map[string]string, []map[string]string, *v3.Point) { // Each row will have a value and a timestamp, and an optional list of label values // example: {Timestamp: ..., Value: ...} diff --git a/pkg/query-service/app/parser.go b/pkg/query-service/app/parser.go index 9bdde09be6..bbf97c7adf 100644 --- a/pkg/query-service/app/parser.go +++ b/pkg/query-service/app/parser.go @@ -846,15 +846,41 @@ func parseQBFilterSuggestionsRequest(r *http.Request) ( return nil, model.BadRequest(err) } - limit := baseconstants.DefaultFilterSuggestionsLimit - limitStr := r.URL.Query().Get("limit") - if len(limitStr) > 0 { - limit, err := strconv.Atoi(limitStr) - if err != nil || limit < 1 { - return nil, model.BadRequest(fmt.Errorf( - "invalid limit: %s", limitStr, - )) + parsePositiveIntQP := func( + queryParam string, defaultValue uint64, maxValue uint64, + ) (uint64, *model.ApiError) { + value := defaultValue + + qpValue := r.URL.Query().Get(queryParam) + if len(qpValue) > 0 { + value, err := strconv.Atoi(qpValue) + + if err != nil || value < 1 || value > int(maxValue) { + return 0, model.BadRequest(fmt.Errorf( + "invalid %s: %s", queryParam, qpValue, + )) + } } + + return value, nil + } + + attributesLimit, err := parsePositiveIntQP( + "attributesLimit", + baseconstants.DefaultFilterSuggestionsAttributesLimit, + baseconstants.MaxFilterSuggestionsAttributesLimit, + ) + if err != nil { + return nil, err + } + + examplesLimit, err := parsePositiveIntQP( + "examplesLimit", + baseconstants.DefaultFilterSuggestionsExamplesLimit, + baseconstants.MaxFilterSuggestionsExamplesLimit, + ) + if err != nil { + return nil, err } var existingFilter *v3.FilterSet @@ -875,10 +901,11 @@ func parseQBFilterSuggestionsRequest(r *http.Request) ( searchText := r.URL.Query().Get("searchText") return &v3.QBFilterSuggestionsRequest{ - DataSource: dataSource, - Limit: limit, - SearchText: searchText, - ExistingFilter: existingFilter, + DataSource: dataSource, + SearchText: searchText, + ExistingFilter: existingFilter, + AttributesLimit: attributesLimit, + ExamplesLimit: examplesLimit, }, nil } diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 4b5134c6ee..8c8a038f2f 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -417,4 +417,7 @@ var TracesListViewDefaultSelectedColumns = []v3.AttributeKey{ }, } -const DefaultFilterSuggestionsLimit = 100 +const DefaultFilterSuggestionsAttributesLimit = 50 +const MaxFilterSuggestionsAttributesLimit = 100 +const DefaultFilterSuggestionsExamplesLimit = 2 +const MaxFilterSuggestionsExamplesLimit = 10 diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index b0e786a6d6..0f04375198 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -253,10 +253,11 @@ type FilterAttributeKeyRequest struct { } type QBFilterSuggestionsRequest struct { - DataSource DataSource `json:"dataSource"` - SearchText string `json:"searchText"` - Limit int `json:"limit"` - ExistingFilter *FilterSet `json:"existing_filter"` + DataSource DataSource `json:"dataSource"` + SearchText string `json:"searchText"` + ExistingFilter *FilterSet `json:"existingFilter"` + AttributesLimit uint64 `json:"attributesLimit"` + ExamplesLimit uint64 `json:"examplesLimit"` } type QBFilterSuggestionsResponse struct { diff --git a/pkg/query-service/tests/integration/filter_suggestions_test.go b/pkg/query-service/tests/integration/filter_suggestions_test.go index 8c379b1c10..6859a6ac2f 100644 --- a/pkg/query-service/tests/integration/filter_suggestions_test.go +++ b/pkg/query-service/tests/integration/filter_suggestions_test.go @@ -19,6 +19,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/model" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/utils" + "go.uber.org/zap" ) // If no data has been received yet, filter suggestions should contain @@ -59,7 +60,9 @@ func TestLogsFilterSuggestionsWithoutExistingFilter(t *testing.T) { testAttribValue := "test-container" tb.mockAttribKeysQueryResponse([]v3.AttributeKey{testAttrib}) - tb.mockAttribValuesQueryResponse(testAttrib, []string{testAttribValue}) + tb.mockAttribValuesQueryResponse( + []v3.AttributeKey{testAttrib}, [][]string{{testAttribValue}}, + ) suggestionsQueryParams := map[string]string{} suggestionsResp := tb.GetQBFilterSuggestionsForLogs(suggestionsQueryParams) @@ -71,6 +74,7 @@ func TestLogsFilterSuggestionsWithoutExistingFilter(t *testing.T) { )) require.Greater(len(suggestionsResp.ExampleQueries), 0) + require.True(slices.ContainsFunc( suggestionsResp.ExampleQueries, func(q v3.FilterSet) bool { return slices.ContainsFunc(q.Items, func(i v3.FilterItem) bool { @@ -113,7 +117,10 @@ func TestLogsFilterSuggestionsWithExistingFilter(t *testing.T) { } tb.mockAttribKeysQueryResponse([]v3.AttributeKey{testAttrib, testFilterAttrib}) - tb.mockAttribValuesQueryResponse(testAttrib, []string{testAttribValue}) + tb.mockAttribValuesQueryResponse( + []v3.AttributeKey{testAttrib, testFilterAttrib}, + [][]string{{testAttribValue}, {testFilterAttribValue}}, + ) testFilterJson, err := json.Marshal(testFilter) require.Nil(err, "couldn't serialize existing filter to JSON") @@ -152,7 +159,7 @@ func (tb *FilterSuggestionsTestBed) mockAttribKeysQueryResponse( tb.mockClickhouse.ExpectQuery( "select.*from.*signoz_logs.distributed_tag_attributes.*", ).WithArgs( - constants.DefaultFilterSuggestionsLimit, + constants.DefaultFilterSuggestionsAttributesLimit, ).WillReturnRows( mockhouse.NewRows(cols, values), ) @@ -169,22 +176,30 @@ func (tb *FilterSuggestionsTestBed) mockAttribKeysQueryResponse( // Mocks response for CH queries made by reader.GetLogAttributeValues func (tb *FilterSuggestionsTestBed) mockAttribValuesQueryResponse( - expectedAttrib v3.AttributeKey, - stringValuesToReturn []string, + expectedAttribs []v3.AttributeKey, + stringValuesToReturn [][]string, ) { - cols := []mockhouse.ColumnType{} - cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "stringTagValue"}) + resultCols := []mockhouse.ColumnType{ + {Type: "String", Name: "tagKey"}, + {Type: "String", Name: "stringTagValue"}, + {Type: "Nullable(Int64)", Name: "int64TagValue"}, + {Type: "Nullable(Float64)", Name: "float64TagValue"}, + } - values := [][]any{} - for _, v := range stringValuesToReturn { - rowValues := []any{} - rowValues = append(rowValues, v) - values = append(values, rowValues) + expectedAttribKeysInQuery := []string{} + mockResultRows := [][]any{} + for idx, attrib := range expectedAttribs { + expectedAttribKeysInQuery = append(expectedAttribKeysInQuery, attrib.Key) + for _, stringTagValue := range stringValuesToReturn[idx] { + mockResultRows = append(mockResultRows, []any{ + attrib.Key, stringTagValue, nil, nil, + }) + } } tb.mockClickhouse.ExpectQuery( - "select distinct.*stringTagValue.*from.*signoz_logs.distributed_tag_attributes.*", - ).WithArgs(string(expectedAttrib.Key), v3.TagType(expectedAttrib.Type), 1).WillReturnRows(mockhouse.NewRows(cols, values)) + "select.*tagKey.*stringTagValue.*int64TagValue.*float64TagValue.*distributed_tag_attributes.*tagKey.*in.*", + ).WithArgs(expectedAttribKeysInQuery).WillReturnRows(mockhouse.NewRows(resultCols, mockResultRows)) } type FilterSuggestionsTestBed struct { @@ -244,6 +259,13 @@ func NewFilterSuggestionsTestBed(t *testing.T) *FilterSuggestionsTestBed { t.Fatalf("could not create a test user: %v", apiErr) } + logger := zap.NewExample() + originalLogger := zap.L() + zap.ReplaceGlobals(logger) + t.Cleanup(func() { + zap.ReplaceGlobals(originalLogger) + }) + return &FilterSuggestionsTestBed{ t: t, testUser: user,