adapt log body filtering to allow for text after the redacted sequence

Signed-off-by: Moritz Wiesinger <[email protected]>
Dynatrace · Nov 11, 2024 · cf38723 · cf38723
1 parent 36dc781
commit cf38723
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 14 deletions.
diff --git a/config_examples/masking_logbody.yaml b/config_examples/masking_logbody.yaml
@@ -16,16 +16,16 @@ processors:
     log_statements:
       - context: log
         statements:
-          - replace_pattern(body, "3\\s*[47](\\s*[0-9]){9}((\\s*[0-9]){4})$", "**** $$2")
-          - replace_pattern(body, "(5[1-5]([0-9]){2}|222[1-9]|22[3-9]\\d|2[3-6]\\d{2}|27[0-1]\\d|2720)(\\s*[0-9]){8}\\s*([0-9]{4})$", "**** $$4")
-          - replace_pattern(body, "4(\\s*[0-9]){8,14}\\s*(([0-9]\\s*){4})$", "**** $$2")
+          - replace_pattern(body, "(^|\\s)3\\s*[47](\\s*[0-9]){9}((\\s*[0-9]){4})($|\\s)", "$$1**** $$3$$5")
+          - replace_pattern(body, "(^|\\s)(5[1-5]([0-9]){2}|222[1-9]|22[3-9]\\d|2[3-6]\\d{2}|27[0-1]\\d|2720)(\\s*[0-9]){8}\\s*([0-9]{4})($|\\s)", "$$1**** $$5$$6")
+          - replace_pattern(body, "(^|\\s)4(\\s*[0-9]){8,14}\\s*(([0-9]\\s*?){4})($|\\s)", "$$1**** $$3$$5")
   transform/iban:
     error_mode: ignore
     trace_statements:
     log_statements:
       - context: log
         statements:
-          - replace_pattern(body, "([A-Z]{2})[0-9]{2}((\\s*[A-Z0-9]){4,26})\\s*((\\s*[A-Z0-9]){4})$", "$$1 **** $$4")
+          - replace_pattern(body, "(^|\\s)([A-Z]{2})[0-9]{2}((\\s*[A-Z0-9]){4,26})\\s*((\\s*[A-Z0-9]){4})($|\\s)", "$$1$$2 **** $$5$$7")
 
 
 service:

diff --git a/internal/testbed/integration/filtering_test.go b/internal/testbed/integration/filtering_test.go
@@ -160,46 +160,60 @@ func TestFilteringLogBody(t *testing.T) {
 		"card_master_spaces1 2367 8901 2345 6789",
 		"card_master_spaces2 5105 1051 0510 5100",
 		"card_master_spaces3 2720 1051 0510 5100",
+		"card_master_spaces4 2720 1051 0520 5100 some text after",
 		"card_master_no_spaces1 2367890123456789",
 		"card_master_no_spaces2 5105105105105100",
 		"card_master_no_spaces3 2720105105105100",
+		"card_master_no_spaces4 2720145107105100 testy test",
 		"card_visa_spaces1 4539 1488 0343 6467",
 		"card_visa_spaces2 4539 1488 0343 6",
 		"card_visa_spaces3 4539 1488 0343 6467 234",
+		"card_visa_spaces4 4539 1498 0343 6457 234 testy test",
 		"card_visa_no_spaces1 4539148803436467",
 		"card_visa_no_spaces2 4539148803436",
 		"card_visa_no_spaces3 4539148803436467234",
+		"card_visa_no_spaces4 4539148812336467234 asdf asdf",
 		"card_amex_spaces1 3714 496353 98431",
 		"card_amex_spaces2 3487 344936 71000",
 		"card_amex_spaces3 3782 822463 10005",
+		"card_amex_spaces4 3782 822463 12305 testy test.",
 		"card_amex_no_spaces1 371449635398431",
 		"card_amex_no_spaces2 348734493671000",
 		"card_amex_no_spaces3 378282246310005",
+		"card_amex_no_spaces4 378282243210005 some postfix",
 		"safe_attribute1 371",
 		"safe_attribute2 37810005",
+		"safe_attribute3 987346 some postfix test",
 	}
 
 	logsCCFiltered := []string{
 		"card_master_spaces1 **** 6789",
 		"card_master_spaces2 **** 5100",
 		"card_master_spaces3 **** 5100",
+		"card_master_spaces4 **** 5100 some text after",
 		"card_master_no_spaces1 **** 6789",
 		"card_master_no_spaces2 **** 5100",
 		"card_master_no_spaces3 **** 5100",
+		"card_master_no_spaces4 **** 5100 testy test",
 		"card_visa_spaces1 **** 6467",
 		"card_visa_spaces2 **** 343 6",
 		"card_visa_spaces3 **** 7 234",
+		"card_visa_spaces4 **** 7 234 testy test",
 		"card_visa_no_spaces1 **** 6467",
 		"card_visa_no_spaces2 **** 3436",
 		"card_visa_no_spaces3 **** 7234",
+		"card_visa_no_spaces4 **** 7234 asdf asdf",
 		"card_amex_spaces1 **** 8431",
 		"card_amex_spaces2 **** 1000",
 		"card_amex_spaces3 **** 0005",
+		"card_amex_spaces4 **** 2305 testy test.",
 		"card_amex_no_spaces1 **** 8431",
 		"card_amex_no_spaces2 **** 1000",
 		"card_amex_no_spaces3 **** 0005",
+		"card_amex_no_spaces4 **** 0005 some postfix",
 		"safe_attribute1 371",
 		"safe_attribute2 37810005",
+		"safe_attribute3 987346 some postfix test",
 	}
 
 	logsIbanPlain := []string{
@@ -209,25 +223,25 @@ func TestFilteringLogBody(t *testing.T) {
 		"iban4 IT60 X054 2811 1010 0000 0123 456",
 		"iban5 NL91 ABNA 0417 1643 00",
 		"iban6 BE71 0961 2345 6769",
-		"iban7 AT48 3200 0000 0123 4568",
+		"iban7 AT48 3200 0000 0123 4568 testy test",
 		"iban8 SE72 8000 0810 0340 0978 3242",
 		"iban9 PL61 1090 1014 0000 0712 1981 2874",
 		"iban10 GB29 NWBK 6016 1331 9268 19",
-		"iban11 AL47 2121 1009 0000 0002 3569 8741",
+		"iban11 AL47 2121 1009 0000 0002 3569 8741 asdf asdf",
 		"iban12 CY17 0020 0128 0000 0012 0052 7600",
 		"iban13 KW81 CBKU 0000 0000 0000 1234 5601 01",
 		"iban14 LU28 0019 4006 4475 0000",
 		"iban15 NO93 8601 1117 947",
 		"iban16 DE89370400440532013000",
-		"iban17 FR1420041010050500013M02606",
+		"iban17 FR1420041010050500013M02606 test222",
 		"iban18 ES9121000418450200051332",
 		"iban19 IT60X0542811101000000123456",
-		"iban20 NL91ABNA0417164300",
+		"iban20 NL91ABNA0417164300 text text",
 		"iban21 BE71096123456769",
 		"iban22 AT483200000001234568",
 		"iban23 SE7280000810034009783242",
 		"iban24 PL61109010140000071219812874",
-		"iban25 GB29NWBK60161331926819",
+		"iban25 GB29NWBK60161331926819 postfix text",
 		"iban26 AL47212110090000000235698741",
 		"iban27 CY17002001280000001200527600",
 		"iban28 KW81CBKU0000000000001234560101",
@@ -243,25 +257,25 @@ func TestFilteringLogBody(t *testing.T) {
 		"iban4 IT **** 3 456",
 		"iban5 NL **** 43 00",
 		"iban6 BE **** 6769",
-		"iban7 AT **** 4568",
+		"iban7 AT **** 4568 testy test",
 		"iban8 SE **** 3242",
 		"iban9 PL **** 2874",
 		"iban10 GB **** 68 19",
-		"iban11 AL **** 8741",
+		"iban11 AL **** 8741 asdf asdf",
 		"iban12 CY **** 7600",
 		"iban13 KW **** 01 01",
 		"iban14 LU **** 0000",
 		"iban15 NO **** 7 947",
 		"iban16 DE **** 3000",
-		"iban17 FR **** 2606",
+		"iban17 FR **** 2606 test222",
 		"iban18 ES **** 1332",
 		"iban19 IT **** 3456",
-		"iban20 NL **** 4300",
+		"iban20 NL **** 4300 text text",
 		"iban21 BE **** 6769",
 		"iban22 AT **** 4568",
 		"iban23 SE **** 3242",
 		"iban24 PL **** 2874",
-		"iban25 GB **** 6819",
+		"iban25 GB **** 6819 postfix text",
 		"iban26 AL **** 8741",
 		"iban27 CY **** 7600",
 		"iban28 KW **** 0101",