From 9c8e75036efc6284834f448787a000b11751b7f6 Mon Sep 17 00:00:00 2001 From: makeavish Date: Mon, 20 May 2024 09:48:49 +0530 Subject: [PATCH 1/2] chore: ignore tag attributes which has an equivalent fixed column --- .../app/clickhouseReader/reader.go | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index fcc2efeb15..c2057295b7 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -4828,6 +4828,11 @@ func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req if err := rows.Scan(&tagKey, &tagType, &dataType, &isColumn); err != nil { return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) } + + if ignoreTagKeysWhichAreIndexed(tagKey) { + continue + } + // TODO: Remove this once the column name are updated in the table tagKey = tempHandleFixedColumns(tagKey) key := v3.AttributeKey{ @@ -4869,6 +4874,11 @@ func (r *ClickHouseReader) GetTraceAttributeKeys(ctx context.Context, req *v3.Fi if err := rows.Scan(&tagKey, &tagType, &dataType, &isColumn); err != nil { return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) } + + if ignoreTagKeysWhichAreIndexed(tagKey) { + continue + } + // TODO: Remove this once the column name are updated in the table tagKey = tempHandleFixedColumns(tagKey) key := v3.AttributeKey{ @@ -4895,6 +4905,33 @@ func tempHandleFixedColumns(tagKey string) string { return tagKey } +// ignoreTagKeysWhichAreIndexed ignores tag attributes which has an equivalent fixed column +func ignoreTagKeysWhichAreIndexed(tagKey string) bool { + excludedTagsMap := map[string]bool{ + "db.system": true, + "db.name": true, + "db.operation": true, + "peer.service": true, + "url.full": true, + "http.response.status_code": true, + "http.route": true, + "http.method": true, + "http.request.method": true, + "http.url": true, + "http.status_code": true, + "http.host": true, + "messaging.system": true, + "messaging.operation": true, + "service.name": true, + "rpc.method": true, + "rpc.service": true, + "rpc.system": true, + "rpc.jsonrpc.error_code": true, + "rpc.grpc.status_code": true, + } + return excludedTagsMap[tagKey] +} + func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { var query string From 3170c344dcfbd4acdda96e5d49fbd24e97004a8f Mon Sep 17 00:00:00 2001 From: makeavish Date: Fri, 24 May 2024 23:14:18 +0530 Subject: [PATCH 2/2] chore: use equivalent column attribute --- .../app/clickhouseReader/reader.go | 35 ----- pkg/query-service/app/http_handler.go | 142 ++++++++++++++++++ .../app/traces/v3/query_builder.go | 25 ++- .../app/traces/v3/query_builder_test.go | 22 +-- 4 files changed, 165 insertions(+), 59 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 302823438f..8ec45be168 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -4840,10 +4840,6 @@ func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) } - if ignoreTagKeysWhichAreIndexed(tagKey) { - continue - } - // TODO: Remove this once the column name are updated in the table tagKey = tempHandleFixedColumns(tagKey) key := v3.AttributeKey{ @@ -4886,10 +4882,6 @@ func (r *ClickHouseReader) GetTraceAttributeKeys(ctx context.Context, req *v3.Fi return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) } - if ignoreTagKeysWhichAreIndexed(tagKey) { - continue - } - // TODO: Remove this once the column name are updated in the table tagKey = tempHandleFixedColumns(tagKey) key := v3.AttributeKey{ @@ -4916,33 +4908,6 @@ func tempHandleFixedColumns(tagKey string) string { return tagKey } -// ignoreTagKeysWhichAreIndexed ignores tag attributes which has an equivalent fixed column -func ignoreTagKeysWhichAreIndexed(tagKey string) bool { - excludedTagsMap := map[string]bool{ - "db.system": true, - "db.name": true, - "db.operation": true, - "peer.service": true, - "url.full": true, - "http.response.status_code": true, - "http.route": true, - "http.method": true, - "http.request.method": true, - "http.url": true, - "http.status_code": true, - "http.host": true, - "messaging.system": true, - "messaging.operation": true, - "service.name": true, - "rpc.method": true, - "rpc.service": true, - "rpc.system": true, - "rpc.jsonrpc.error_code": true, - "rpc.grpc.status_code": true, - } - return excludedTagsMap[tagKey] -} - func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { var query string diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 4d63c887fd..3255fe0af4 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -3106,6 +3106,148 @@ func (aH *APIHandler) getSpanKeysV3(ctx context.Context, queryRangeParams *v3.Qu Key: "timestamp", IsColumn: true, } + spanKeys["http.url"] = v3.AttributeKey{ + Key: "httpUrl", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["db.system"] = v3.AttributeKey{ + Key: "dbSystem", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["db.name"] = v3.AttributeKey{ + Key: "dbName", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["db.operation"] = v3.AttributeKey{ + Key: "dbOperation", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["peer.service"] = v3.AttributeKey{ + Key: "peerService", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["url.full"] = v3.AttributeKey{ + Key: "httpUrl", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["http.response.status_code"] = v3.AttributeKey{ + Key: "responseStatusCode", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["http.route"] = v3.AttributeKey{ + Key: "httpRoute", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["http.method"] = v3.AttributeKey{ + Key: "httpMethod", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["http.request.method"] = v3.AttributeKey{ + Key: "httpMethod", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["http.status_code"] = v3.AttributeKey{ + Key: "responseStatusCode", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["http.host"] = v3.AttributeKey{ + Key: "httpHost", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["http.host"] = v3.AttributeKey{ + Key: "httpHost", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["http.request.header.host"] = v3.AttributeKey{ + Key: "httpHost", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["server.address"] = v3.AttributeKey{ + Key: "httpHost", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["client.address"] = v3.AttributeKey{ + Key: "httpHost", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["messaging.system"] = v3.AttributeKey{ + Key: "msgSystem", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["messaging.operation"] = v3.AttributeKey{ + Key: "msgOperation", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["service.name"] = v3.AttributeKey{ + Key: "serviceName", + IsColumn: true, + } + spanKeys["rpc.method"] = v3.AttributeKey{ + Key: "rpcMethod", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["rpc.service"] = v3.AttributeKey{ + Key: "rpcService", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["rpc.system"] = v3.AttributeKey{ + Key: "rpcSystem", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["rpc.jsonrpc.error_code"] = v3.AttributeKey{ + Key: "responseStatusCode", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } + spanKeys["rpc.grpc.status_code"] = v3.AttributeKey{ + Key: "responseStatusCode", + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + DataType: v3.AttributeKeyDataTypeString, + } return spanKeys, nil } } diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index efbc4d8872..a380c051df 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -85,18 +85,17 @@ func getClickhouseTracesColumnDataTypeAndType(key v3.AttributeKey) (v3.Attribute } func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) v3.AttributeKey { - if key.Type == "" || key.DataType == "" { - // check if the key is present in the keys map - if existingKey, ok := keys[key.Key]; ok { - key.IsColumn = existingKey.IsColumn - key.Type = existingKey.Type - key.DataType = existingKey.DataType - } else { // if not present then set the default values - key.Type = v3.AttributeKeyTypeTag - key.DataType = v3.AttributeKeyDataTypeString - key.IsColumn = false - return key - } + // check if the key is present in the keys map + if existingKey, ok := keys[key.Key]; ok { + key.Key = existingKey.Key + key.IsColumn = existingKey.IsColumn + key.Type = existingKey.Type + key.DataType = existingKey.DataType + } else if key.Type == "" || key.DataType == "" { // if not present then set the default values + key.Type = v3.AttributeKeyTypeTag + key.DataType = v3.AttributeKeyDataTypeString + key.IsColumn = false + return key } return key } @@ -263,7 +262,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str queryTmpl = "SELECT now() as ts," // step or aggregate interval is whole time period in case of table panel - step = (end*getZerosForEpochNano(end) - start*getZerosForEpochNano(start))/1000000000 + step = (end*getZerosForEpochNano(end) - start*getZerosForEpochNano(start)) / 1000000000 } else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue { // Select the aggregate value for interval queryTmpl = diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index b4b1a2574c..5395fb8624 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -631,12 +631,12 @@ var testBuildTracesQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", }, TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name'])))" + + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name)))" + " as value from signoz_traces.distributed_signoz_index_v2 where" + " (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by value DESC", PanelType: v3.PanelTypeGraph, @@ -915,7 +915,7 @@ var testBuildTracesQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", Having: []v3.Having{ @@ -927,7 +927,7 @@ var testBuildTracesQueryData = []struct { }, }, TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" + + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " group by ts having value > 10 order by value DESC", PanelType: v3.PanelTypeGraph, @@ -939,7 +939,7 @@ var testBuildTracesQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -957,7 +957,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from " + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by value DESC", + "AND stringTagMap['method'] = 'GET' AND name != '' group by ts having value > 10 order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -983,7 +983,7 @@ var testBuildTracesQueryData = []struct { }, }, TableName: "signoz_traces.distributed_signoz_index_v2", - ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" + + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by value DESC", PanelType: v3.PanelTypeGraph, @@ -995,7 +995,7 @@ var testBuildTracesQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -1013,7 +1013,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10", + "AND stringTagMap['method'] = 'GET' AND name != '' group by ts having value > 10", PanelType: v3.PanelTypeValue, }, { @@ -1204,7 +1204,7 @@ var testPrepTracesQueryData = []struct { GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, }, ExpectedQuery: "SELECT `method` from (SELECT stringTagMap['method'] as `method`," + - " toFloat64(count(distinct(stringTagMap['name']))) as value from signoz_traces.distributed_signoz_index_v2" + + " toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2" + " where (timestamp >= '1680066360000000000' AND timestamp <= '1680066420000000000') AND" + " stringTagMap['method'] = 'GET' AND has(stringTagMap, 'method') group by `method` order by value DESC) LIMIT 10", Keys: map[string]v3.AttributeKey{"name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, @@ -1332,7 +1332,7 @@ var testPrepTracesQueryData = []struct { End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", - AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{