Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support for case insensitive for contains and like queries #6045

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

feat: support for case insensitive for contains and like queries #6045

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions pkg/query-service/app/logs/v4/query_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ var logOperators = map[v3.FilterOperator]string{
v3.FilterOperatorLessThanOrEq: "<=",
v3.FilterOperatorGreaterThan: ">",
v3.FilterOperatorGreaterThanOrEq: ">=",
v3.FilterOperatorLike: "LIKE",
v3.FilterOperatorNotLike: "NOT LIKE",
v3.FilterOperatorContains: "LIKE",
v3.FilterOperatorNotContains: "NOT LIKE",
v3.FilterOperatorLike: "ILIKE",
v3.FilterOperatorNotLike: "NOT ILIKE",
v3.FilterOperatorContains: "ILIKE",
v3.FilterOperatorNotContains: "NOT ILIKE",
v3.FilterOperatorRegex: "match(%s, %s)",
v3.FilterOperatorNotRegex: "NOT match(%s, %s)",
v3.FilterOperatorIn: "IN",
Expand Down Expand Up @@ -150,6 +150,7 @@ func buildAttributeFilter(item v3.FilterItem) (string, error) {
val := utils.QuoteEscapedStringForContains(fmt.Sprintf("%s", item.Value))
// for body the contains is case insensitive
if keyName == BODY {
logsOp = strings.Replace(logsOp, "ILIKE", "LIKE", 1) // removing i from ilike and not ilike
return fmt.Sprintf("lower(%s) %s lower('%%%s%%')", keyName, logsOp, val), nil
} else {
return fmt.Sprintf("%s %s '%%%s%%'", keyName, logsOp, val), nil
Expand All @@ -158,6 +159,7 @@ func buildAttributeFilter(item v3.FilterItem) (string, error) {
// for body use lower for like and ilike
val := utils.QuoteEscapedString(fmt.Sprintf("%s", item.Value))
if keyName == BODY {
logsOp = strings.Replace(logsOp, "ILIKE", "LIKE", 1) // removing i from ilike and not ilike
return fmt.Sprintf("lower(%s) %s lower('%s')", keyName, logsOp, val), nil
} else {
return fmt.Sprintf("%s %s '%s'", keyName, logsOp, val), nil
Expand Down
6 changes: 3 additions & 3 deletions pkg/query-service/app/logs/v4/query_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ func Test_buildAttributeFilter(t *testing.T) {
Value: "test",
},
},
want: "resources_string['service.name'] LIKE '%test%'",
want: "resources_string['service.name'] ILIKE '%test%'",
},
{
name: "build attribute filter contains- body",
Expand Down Expand Up @@ -280,7 +280,7 @@ func Test_buildAttributeFilter(t *testing.T) {
Value: "test%",
},
},
want: "resources_string['service.name'] LIKE 'test%'",
want: "resources_string['service.name'] ILIKE 'test%'",
},
{
name: "build attribute filter like-body",
Expand Down Expand Up @@ -956,7 +956,7 @@ func TestPrepareLogsQuery(t *testing.T) {
},
want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string from " +
"signoz_logs.distributed_logs_v2 where attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND " +
"(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE simpleJSONExtractString(labels, 'service.name') LIKE '%app%' AND labels like '%service.name%app%' AND ",
"(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE simpleJSONExtractString(lower(labels), 'service.name') LIKE '%app%' AND lower(labels) like '%service.name%app%' AND ",
},
{
name: "Live Tail Query W/O filter",
Expand Down
63 changes: 49 additions & 14 deletions pkg/query-service/app/logs/v4/resource_query_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,37 @@ import (
"go.signoz.io/signoz/pkg/query-service/utils"
)

var resourceLogOperators = map[v3.FilterOperator]string{
v3.FilterOperatorEqual: "=",
v3.FilterOperatorNotEqual: "!=",
v3.FilterOperatorLessThan: "<",
v3.FilterOperatorLessThanOrEq: "<=",
v3.FilterOperatorGreaterThan: ">",
v3.FilterOperatorGreaterThanOrEq: ">=",
v3.FilterOperatorLike: "LIKE",
v3.FilterOperatorNotLike: "NOT LIKE",
v3.FilterOperatorContains: "LIKE",
v3.FilterOperatorNotContains: "NOT LIKE",
v3.FilterOperatorRegex: "match(%s, %s)",
v3.FilterOperatorNotRegex: "NOT match(%s, %s)",
v3.FilterOperatorIn: "IN",
v3.FilterOperatorNotIn: "NOT IN",
v3.FilterOperatorExists: "mapContains(%s_%s, '%s')",
v3.FilterOperatorNotExists: "not mapContains(%s_%s, '%s')",
}

// buildResourceFilter builds a clickhouse filter string for resource labels
func buildResourceFilter(logsOp string, key string, op v3.FilterOperator, value interface{}) string {
// for all operators except contains and like
searchKey := fmt.Sprintf("simpleJSONExtractString(labels, '%s')", key)

// for contains and like it will be case insensitive
lowerSearchKey := fmt.Sprintf("simpleJSONExtractString(lower(labels), '%s')", key)

chFmtVal := utils.ClickHouseFormattedValue(value)

lowerValue := strings.ToLower(fmt.Sprintf("%s", value))

switch op {
case v3.FilterOperatorExists:
return fmt.Sprintf("simpleJSONHas(labels, '%s')", key)
Expand All @@ -24,20 +49,20 @@ func buildResourceFilter(logsOp string, key string, op v3.FilterOperator, value
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
// this is required as clickhouseFormattedValue add's quotes to the string
// we also want to treat %, _ as literals for contains
escapedStringValue := utils.QuoteEscapedStringForContains(fmt.Sprintf("%s", value))
return fmt.Sprintf("%s %s '%%%s%%'", searchKey, logsOp, escapedStringValue)
escapedStringValue := utils.QuoteEscapedStringForContains(lowerValue)
return fmt.Sprintf("%s %s '%%%s%%'", lowerSearchKey, logsOp, escapedStringValue)
case v3.FilterOperatorLike, v3.FilterOperatorNotLike:
// this is required as clickhouseFormattedValue add's quotes to the string
escapedStringValue := utils.QuoteEscapedString(fmt.Sprintf("%s", value))
return fmt.Sprintf("%s %s '%s'", searchKey, logsOp, escapedStringValue)
escapedStringValue := utils.QuoteEscapedString(lowerValue)
return fmt.Sprintf("%s %s '%s'", lowerSearchKey, logsOp, escapedStringValue)
default:
return fmt.Sprintf("%s %s %s", searchKey, logsOp, chFmtVal)
}
}

// buildIndexFilterForInOperator builds a clickhouse filter string for in operator
// example:= x in a,b,c = (labels like '%x%a%' or labels like '%"x":"b"%' or labels like '%"x"="c"%')
// example:= x nin a,b,c = (labels nlike '%x%a%' AND labels nlike '%"x"="b"' AND labels nlike '%"x"="c"%')
// example:= x in a,b,c = (labels like '%"x"%"a"%' or labels like '%"x":"b"%' or labels like '%"x"="c"%')
// example:= x nin a,b,c = (labels nlike '%"x"%"a"%' AND labels nlike '%"x"="b"' AND labels nlike '%"x"="c"%')
func buildIndexFilterForInOperator(key string, op v3.FilterOperator, value interface{}) string {
conditions := []string{}
separator := " OR "
Expand Down Expand Up @@ -77,24 +102,34 @@ func buildIndexFilterForInOperator(key string, op v3.FilterOperator, value inter

// buildResourceIndexFilter builds a clickhouse filter string for resource labels
// example:= x like '%john%' = labels like '%x%john%'
// we have two indexes for resource attributes one is lower and one is normal.
// for all operators other then like/contains we will use normal index
// for like/contains we will use lower index
// we can use lower index for =, in etc but it's difficult to do it for !=, NIN etc
// if as x != "ABC" we cannot predict something like "not lower(labels) like '%%x%%abc%%'". It has it be "not lower(labels) like '%%x%%ABC%%'"
func buildResourceIndexFilter(key string, op v3.FilterOperator, value interface{}) string {
// not using clickhouseFormattedValue as we don't wan't the quotes
strVal := fmt.Sprintf("%s", value)
formattedValueEscapedForContains := utils.QuoteEscapedStringForContains(strVal)
formattedValueEscapedForContains := strings.ToLower(utils.QuoteEscapedStringForContains(strVal))
formattedValueEscaped := utils.QuoteEscapedString(strVal)

// add index filters
switch op {
case v3.FilterOperatorContains:
return fmt.Sprintf("labels like '%%%s%%%s%%'", key, formattedValueEscapedForContains)
return fmt.Sprintf("lower(labels) like '%%%s%%%s%%'", key, formattedValueEscapedForContains)
case v3.FilterOperatorNotContains:
return fmt.Sprintf("labels not like '%%%s%%%s%%'", key, formattedValueEscapedForContains)
case v3.FilterOperatorLike, v3.FilterOperatorEqual:
return fmt.Sprintf("lower(labels) not like '%%%s%%%s%%'", key, formattedValueEscapedForContains)
case v3.FilterOperatorLike:
return fmt.Sprintf("lower(labels) like '%%%s%%%s%%'", key, formattedValueEscaped)
case v3.FilterOperatorNotLike:
return fmt.Sprintf("lower(labels) not like '%%%s%%%s%%'", key, formattedValueEscaped)
case v3.FilterOperatorEqual:
return fmt.Sprintf("labels like '%%%s%%%s%%'", key, formattedValueEscaped)
case v3.FilterOperatorNotLike, v3.FilterOperatorNotEqual:
case v3.FilterOperatorNotEqual:
return fmt.Sprintf("labels not like '%%%s%%%s%%'", key, formattedValueEscaped)
case v3.FilterOperatorNotRegex:
return fmt.Sprintf("labels not like '%%%s%%'", key)
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex:
// don't try to do anything for regex.
return ""
case v3.FilterOperatorIn, v3.FilterOperatorNotIn:
return buildIndexFilterForInOperator(key, op, value)
default:
Expand Down Expand Up @@ -137,7 +172,7 @@ func buildResourceFiltersFromFilterItems(fs *v3.FilterSet) ([]string, error) {
}
}

if logsOp, ok := logOperators[op]; ok {
if logsOp, ok := resourceLogOperators[op]; ok {
// the filter
if resourceFilter := buildResourceFilter(logsOp, keyName, op, value); resourceFilter != "" {
conditions = append(conditions, resourceFilter)
Expand Down
20 changes: 10 additions & 10 deletions pkg/query-service/app/logs/v4/resource_query_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func Test_buildResourceFilter(t *testing.T) {
op: v3.FilterOperatorContains,
value: "Application%_",
},
want: `simpleJSONExtractString(labels, 'service.name') LIKE '%Application\%\_%'`,
want: `simpleJSONExtractString(lower(labels), 'service.name') LIKE '%application\%\_%'`,
},
{
name: "test eq",
Expand Down Expand Up @@ -83,7 +83,7 @@ func Test_buildResourceFilter(t *testing.T) {
op: v3.FilterOperatorLike,
value: "Application%_",
},
want: `simpleJSONExtractString(labels, 'service.name') LIKE 'Application%_'`,
want: `simpleJSONExtractString(lower(labels), 'service.name') LIKE 'application%_'`,
},
}
for _, tt := range tests {
Expand Down Expand Up @@ -170,7 +170,7 @@ func Test_buildResourceIndexFilter(t *testing.T) {
op: v3.FilterOperatorContains,
value: "application",
},
want: `labels like '%service.name%application%'`,
want: `lower(labels) like '%service.name%application%'`,
},
{
name: "test not contains",
Expand All @@ -179,7 +179,7 @@ func Test_buildResourceIndexFilter(t *testing.T) {
op: v3.FilterOperatorNotContains,
value: "application",
},
want: `labels not like '%service.name%application%'`,
want: `lower(labels) not like '%service.name%application%'`,
},
{
name: "test contains with % and _",
Expand All @@ -188,7 +188,7 @@ func Test_buildResourceIndexFilter(t *testing.T) {
op: v3.FilterOperatorNotContains,
value: "application%_test",
},
want: `labels not like '%service.name%application\%\_test%'`,
want: `lower(labels) not like '%service.name%application\%\_test%'`,
},
{
name: "test like with % and _",
Expand All @@ -197,7 +197,7 @@ func Test_buildResourceIndexFilter(t *testing.T) {
op: v3.FilterOperatorLike,
value: "application%_test",
},
want: `labels like '%service.name%application%_test%'`,
want: `lower(labels) like '%service.name%application%_test%'`,
},
{
name: "test not regex",
Expand All @@ -206,7 +206,7 @@ func Test_buildResourceIndexFilter(t *testing.T) {
op: v3.FilterOperatorNotRegex,
value: ".*",
},
want: `labels not like '%service.name%'`,
want: ``,
},
{
name: "test in",
Expand Down Expand Up @@ -318,8 +318,8 @@ func Test_buildResourceFiltersFromFilterItems(t *testing.T) {
want: []string{
"simpleJSONExtractString(labels, 'service.name') = 'test'",
"labels like '%service.name%test%'",
"simpleJSONExtractString(labels, 'namespace') LIKE '%test1%'",
"labels like '%namespace%test1%'",
"simpleJSONExtractString(lower(labels), 'namespace') LIKE '%test1%'",
"lower(labels) like '%namespace%test1%'",
},
wantErr: false,
},
Expand Down Expand Up @@ -480,7 +480,7 @@ func Test_buildResourceSubQuery(t *testing.T) {
want: "(SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE " +
"(seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) AND " +
"simpleJSONExtractString(labels, 'service.name') = 'test' AND labels like '%service.name%test%' " +
"AND simpleJSONExtractString(labels, 'namespace') LIKE '%test1%' AND labels like '%namespace%test1%' " +
"AND simpleJSONExtractString(lower(labels), 'namespace') LIKE '%test1%' AND lower(labels) like '%namespace%test1%' " +
"AND (simpleJSONHas(labels, 'cluster.name') AND labels like '%cluster.name%') AND " +
"( (simpleJSONHas(labels, 'host.name') AND labels like '%host.name%') ))",
wantErr: false,
Expand Down
Loading