Shared: Improve sensitive data heuristics

javascript/ql/lib/change-notes/2025-07-11-sensitive-data-heuristics.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The regular expressions in `SensitiveDataHeuristics.qll` have been extended to find more instances of sensitive data such as secrets used in authentication, finance and health information, and device data. The heuristics have also been refined to find fewer false positive matches. This will improve results for queries related to sensitive information.

javascript/ql/lib/semmle/javascript/security/internal/SensitiveDataHeuristics.qll

@@ -54,15 +54,16 @@ module HeuristicNames {
    * Gets a regular expression that identifies strings that may indicate the presence of secret
    * or trusted data.
    */
-  string maybeSecret() { result = "(?is).*((?<!is|is_)secret|(?<!un|un_|is|is_)trusted).*" }
+  string maybeSecret() {
+    result = "(?is).*((?<!is|is_)secret|(?<!un|un_|is|is_)trusted(?!_iter)|confidential).*"
+  }
 
   /**
    * Gets a regular expression that identifies strings that may indicate the presence of
    * user names or other account information.
    */
   string maybeAccountInfo() {
-    result = "(?is).*acc(ou)?nt.*" or
-    result = "(?is).*(puid|user.?name|user.?id|session.?(id|key)).*" or
+    result = "(?is).*(acc(ou)?nt|puid|user.?(name|id)|session.?(id|key)).*" or
     result = "(?s).*([uU]|^|_|[a-z](?=U))([uU][iI][dD]).*"
   }
 
@@ -71,8 +72,9 @@ module HeuristicNames {
    * a password or an authorization key.
    */
   string maybePassword() {
-    result = "(?is).*pass(wd|word|code|.?phrase)(?!.*question).*" or
-    result = "(?is).*(auth(entication|ori[sz]ation)?).?key.*"
+    result =
+      "(?is).*(pass(wd|word|code|.?phrase)(?!.*question)|(auth(entication|ori[sz]ation)?).?key|oauth|"
+        + "api.?(key|token)|([_-]|\\b)mfa([_-]|\\b)).*"
   }
 
   /**
@@ -88,7 +90,7 @@ module HeuristicNames {
   string maybePrivate() {
     result =
       "(?is).*(" +
-        // Inspired by the list on https://cwe.mitre.org/data/definitions/359.html
+        // Inspired by multiple sources including the list on https://cwe.mitre.org/data/definitions/359.html
         // Government identifiers, such as Social Security Numbers
         "social.?security|employer.?identification|national.?insurance|resident.?id|" +
         "passport.?(num|no)|([_-]|\\b)ssn([_-]|\\b)|" +
@@ -100,17 +102,19 @@ module HeuristicNames {
         // Geographic location - where the user is (or was)
         "latitude|longitude|nationality|" +
         // Financial data - such as credit card numbers, salary, bank accounts, and debts
-        "(credit|debit|bank|visa).?(card|num|no|acc(ou)?nt)|acc(ou)?nt.?(no|num|credit)|" +
-        "salary|billing|credit.?(rating|score)|([_-]|\\b)ccn([_-]|\\b)|" +
+        "(credit|debit|bank|visa).?(card|num|no|acc(ou)?nt)|acc(ou)?nt.?(no|num|credit)|routing.?num|"
+        + "salary|billing|beneficiary|credit.?(rating|score)|([_-]|\\b)(ccn|cvv|iban)([_-]|\\b)|" +
         // Communications - e-mail addresses, private e-mail messages, SMS text messages, chat logs, etc.
         // "e(mail|_mail)|" + // this seems too noisy
         // Health - medical conditions, insurance status, prescription records
-        "birth.?da(te|y)|da(te|y).?(of.?)?birth|" +
-        "medical|(health|care).?plan|healthkit|appointment|prescription|" +
+        "birth.?da(te|y)|da(te|y).?(of.?)?birth|gender|([_-]|\\b)sex([_-]|\\b)|" +
+        "medical|(health|care).?plan|healthkit|appointment|prescription|patient.?(id|record)|" +
         "blood.?(type|alcohol|glucose|pressure)|heart.?(rate|rhythm)|body.?(mass|fat)|" +
         "menstrua|pregnan|insulin|inhaler|" +
         // Relationships - work and family
-        "employ(er|ee)|spouse|maiden.?name" +
+        "employ(er|ee)|spouse|maiden.?name|" +
+        // Device information
+        "([_-]|\\b)ip.?addr|mac.?addr|finger.?print" +
         // ---
         ").*"
   }
@@ -144,7 +148,8 @@ module HeuristicNames {
    */
   string notSensitiveRegexp() {
     result =
-      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|certain|concert|secretar|accountant|accountab).*"
+      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|"
+        + "certain|concert|secretar|account(ant|ab|ing|ed)|file|path|([_-]|\\b)url).*"
   }
 
   /**

python/ql/lib/change-notes/2025-07-11-sensitive-data-heuristics.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The regular expressions in `SensitiveDataHeuristics.qll` have been extended to find more instances of sensitive data such as secrets used in authentication, finance and health information, and device data. The heuristics have also been refined to find fewer false positive matches. This will improve results for queries related to sensitive information.

python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll

@@ -54,15 +54,16 @@ module HeuristicNames {
    * Gets a regular expression that identifies strings that may indicate the presence of secret
    * or trusted data.
    */
-  string maybeSecret() { result = "(?is).*((?<!is|is_)secret|(?<!un|un_|is|is_)trusted).*" }
+  string maybeSecret() {
+    result = "(?is).*((?<!is|is_)secret|(?<!un|un_|is|is_)trusted(?!_iter)|confidential).*"
+  }
 
   /**
    * Gets a regular expression that identifies strings that may indicate the presence of
    * user names or other account information.
    */
   string maybeAccountInfo() {
-    result = "(?is).*acc(ou)?nt.*" or
-    result = "(?is).*(puid|user.?name|user.?id|session.?(id|key)).*" or
+    result = "(?is).*(acc(ou)?nt|puid|user.?(name|id)|session.?(id|key)).*" or
     result = "(?s).*([uU]|^|_|[a-z](?=U))([uU][iI][dD]).*"
   }
 
@@ -71,8 +72,9 @@ module HeuristicNames {
    * a password or an authorization key.
    */
   string maybePassword() {
-    result = "(?is).*pass(wd|word|code|.?phrase)(?!.*question).*" or
-    result = "(?is).*(auth(entication|ori[sz]ation)?).?key.*"
+    result =
+      "(?is).*(pass(wd|word|code|.?phrase)(?!.*question)|(auth(entication|ori[sz]ation)?).?key|oauth|"
+        + "api.?(key|token)|([_-]|\\b)mfa([_-]|\\b)).*"
   }
 
   /**
@@ -88,7 +90,7 @@ module HeuristicNames {
   string maybePrivate() {
     result =
       "(?is).*(" +
-        // Inspired by the list on https://cwe.mitre.org/data/definitions/359.html
+        // Inspired by multiple sources including the list on https://cwe.mitre.org/data/definitions/359.html
         // Government identifiers, such as Social Security Numbers
         "social.?security|employer.?identification|national.?insurance|resident.?id|" +
         "passport.?(num|no)|([_-]|\\b)ssn([_-]|\\b)|" +
@@ -100,17 +102,19 @@ module HeuristicNames {
         // Geographic location - where the user is (or was)
         "latitude|longitude|nationality|" +
         // Financial data - such as credit card numbers, salary, bank accounts, and debts
-        "(credit|debit|bank|visa).?(card|num|no|acc(ou)?nt)|acc(ou)?nt.?(no|num|credit)|" +
-        "salary|billing|credit.?(rating|score)|([_-]|\\b)ccn([_-]|\\b)|" +
+        "(credit|debit|bank|visa).?(card|num|no|acc(ou)?nt)|acc(ou)?nt.?(no|num|credit)|routing.?num|"
+        + "salary|billing|beneficiary|credit.?(rating|score)|([_-]|\\b)(ccn|cvv|iban)([_-]|\\b)|" +
         // Communications - e-mail addresses, private e-mail messages, SMS text messages, chat logs, etc.
         // "e(mail|_mail)|" + // this seems too noisy
         // Health - medical conditions, insurance status, prescription records
-        "birth.?da(te|y)|da(te|y).?(of.?)?birth|" +
-        "medical|(health|care).?plan|healthkit|appointment|prescription|" +
+        "birth.?da(te|y)|da(te|y).?(of.?)?birth|gender|([_-]|\\b)sex([_-]|\\b)|" +
+        "medical|(health|care).?plan|healthkit|appointment|prescription|patient.?(id|record)|" +
         "blood.?(type|alcohol|glucose|pressure)|heart.?(rate|rhythm)|body.?(mass|fat)|" +
         "menstrua|pregnan|insulin|inhaler|" +
         // Relationships - work and family
-        "employ(er|ee)|spouse|maiden.?name" +
+        "employ(er|ee)|spouse|maiden.?name|" +
+        // Device information
+        "([_-]|\\b)ip.?addr|mac.?addr|finger.?print" +
         // ---
         ").*"
   }
@@ -144,7 +148,8 @@ module HeuristicNames {
    */
   string notSensitiveRegexp() {
     result =
-      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|certain|concert|secretar|accountant|accountab).*"
+      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|"
+        + "certain|concert|secretar|account(ant|ab|ing|ed)|file|path|([_-]|\\b)url).*"
   }
 
   /**

ruby/ql/lib/change-notes/2025-07-11-sensitive-data-heuristics.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The regular expressions in `SensitiveDataHeuristics.qll` have been extended to find more instances of sensitive data such as secrets used in authentication, finance and health information, and device data. The heuristics have also been refined to find fewer false positive matches. This will improve results for queries related to sensitive information.

ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll

@@ -54,15 +54,16 @@ module HeuristicNames {
    * Gets a regular expression that identifies strings that may indicate the presence of secret
    * or trusted data.
    */
-  string maybeSecret() { result = "(?is).*((?<!is|is_)secret|(?<!un|un_|is|is_)trusted).*" }
+  string maybeSecret() {
+    result = "(?is).*((?<!is|is_)secret|(?<!un|un_|is|is_)trusted(?!_iter)|confidential).*"
+  }
 
   /**
    * Gets a regular expression that identifies strings that may indicate the presence of
    * user names or other account information.
    */
   string maybeAccountInfo() {
-    result = "(?is).*acc(ou)?nt.*" or
-    result = "(?is).*(puid|user.?name|user.?id|session.?(id|key)).*" or
+    result = "(?is).*(acc(ou)?nt|puid|user.?(name|id)|session.?(id|key)).*" or
     result = "(?s).*([uU]|^|_|[a-z](?=U))([uU][iI][dD]).*"
   }
 
@@ -71,8 +72,9 @@ module HeuristicNames {
    * a password or an authorization key.
    */
   string maybePassword() {
-    result = "(?is).*pass(wd|word|code|.?phrase)(?!.*question).*" or
-    result = "(?is).*(auth(entication|ori[sz]ation)?).?key.*"
+    result =
+      "(?is).*(pass(wd|word|code|.?phrase)(?!.*question)|(auth(entication|ori[sz]ation)?).?key|oauth|"
+        + "api.?(key|token)|([_-]|\\b)mfa([_-]|\\b)).*"
   }
 
   /**
@@ -88,7 +90,7 @@ module HeuristicNames {
   string maybePrivate() {
     result =
       "(?is).*(" +
-        // Inspired by the list on https://cwe.mitre.org/data/definitions/359.html
+        // Inspired by multiple sources including the list on https://cwe.mitre.org/data/definitions/359.html
         // Government identifiers, such as Social Security Numbers
         "social.?security|employer.?identification|national.?insurance|resident.?id|" +
         "passport.?(num|no)|([_-]|\\b)ssn([_-]|\\b)|" +
@@ -100,17 +102,19 @@ module HeuristicNames {
         // Geographic location - where the user is (or was)
         "latitude|longitude|nationality|" +
         // Financial data - such as credit card numbers, salary, bank accounts, and debts
-        "(credit|debit|bank|visa).?(card|num|no|acc(ou)?nt)|acc(ou)?nt.?(no|num|credit)|" +
-        "salary|billing|credit.?(rating|score)|([_-]|\\b)ccn([_-]|\\b)|" +
+        "(credit|debit|bank|visa).?(card|num|no|acc(ou)?nt)|acc(ou)?nt.?(no|num|credit)|routing.?num|"
+        + "salary|billing|beneficiary|credit.?(rating|score)|([_-]|\\b)(ccn|cvv|iban)([_-]|\\b)|" +
         // Communications - e-mail addresses, private e-mail messages, SMS text messages, chat logs, etc.
         // "e(mail|_mail)|" + // this seems too noisy
         // Health - medical conditions, insurance status, prescription records
-        "birth.?da(te|y)|da(te|y).?(of.?)?birth|" +
-        "medical|(health|care).?plan|healthkit|appointment|prescription|" +
+        "birth.?da(te|y)|da(te|y).?(of.?)?birth|gender|([_-]|\\b)sex([_-]|\\b)|" +
+        "medical|(health|care).?plan|healthkit|appointment|prescription|patient.?(id|record)|" +
         "blood.?(type|alcohol|glucose|pressure)|heart.?(rate|rhythm)|body.?(mass|fat)|" +
         "menstrua|pregnan|insulin|inhaler|" +
         // Relationships - work and family
-        "employ(er|ee)|spouse|maiden.?name" +
+        "employ(er|ee)|spouse|maiden.?name|" +
+        // Device information
+        "([_-]|\\b)ip.?addr|mac.?addr|finger.?print" +
         // ---
         ").*"
   }
@@ -144,7 +148,8 @@ module HeuristicNames {
    */
   string notSensitiveRegexp() {
     result =
-      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|certain|concert|secretar|accountant|accountab).*"
+      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|"
+        + "certain|concert|secretar|account(ant|ab|ing|ed)|file|path|([_-]|\\b)url).*"
   }
 
   /**

rust/ql/lib/change-notes/2025-07-11-sensitive-data-heuristics.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The regular expressions in `SensitiveDataHeuristics.qll` have been extended to find more instances of sensitive data such as secrets used in authentication, finance and health information, and device data. The heuristics have also been refined to find fewer false positive matches. This will improve results for queries related to sensitive information.

rust/ql/lib/codeql/rust/security/internal/SensitiveDataHeuristics.qll

@@ -54,15 +54,16 @@ module HeuristicNames {
    * Gets a regular expression that identifies strings that may indicate the presence of secret
    * or trusted data.
    */
-  string maybeSecret() { result = "(?is).*((?<!is|is_)secret|(?<!un|un_|is|is_)trusted).*" }
+  string maybeSecret() {
+    result = "(?is).*((?<!is|is_)secret|(?<!un|un_|is|is_)trusted(?!_iter)|confidential).*"
+  }
 
   /**
    * Gets a regular expression that identifies strings that may indicate the presence of
    * user names or other account information.
    */
   string maybeAccountInfo() {
-    result = "(?is).*acc(ou)?nt.*" or
-    result = "(?is).*(puid|user.?name|user.?id|session.?(id|key)).*" or
+    result = "(?is).*(acc(ou)?nt|puid|user.?(name|id)|session.?(id|key)).*" or
     result = "(?s).*([uU]|^|_|[a-z](?=U))([uU][iI][dD]).*"
   }
 
@@ -71,8 +72,9 @@ module HeuristicNames {
    * a password or an authorization key.
    */
   string maybePassword() {
-    result = "(?is).*pass(wd|word|code|.?phrase)(?!.*question).*" or
-    result = "(?is).*(auth(entication|ori[sz]ation)?).?key.*"
+    result =
+      "(?is).*(pass(wd|word|code|.?phrase)(?!.*question)|(auth(entication|ori[sz]ation)?).?key|oauth|"
+        + "api.?(key|token)|([_-]|\\b)mfa([_-]|\\b)).*"
   }
 
   /**
@@ -88,7 +90,7 @@ module HeuristicNames {
   string maybePrivate() {
     result =
       "(?is).*(" +
-        // Inspired by the list on https://cwe.mitre.org/data/definitions/359.html
+        // Inspired by multiple sources including the list on https://cwe.mitre.org/data/definitions/359.html
         // Government identifiers, such as Social Security Numbers
         "social.?security|employer.?identification|national.?insurance|resident.?id|" +
         "passport.?(num|no)|([_-]|\\b)ssn([_-]|\\b)|" +
@@ -100,17 +102,19 @@ module HeuristicNames {
         // Geographic location - where the user is (or was)
         "latitude|longitude|nationality|" +
         // Financial data - such as credit card numbers, salary, bank accounts, and debts
-        "(credit|debit|bank|visa).?(card|num|no|acc(ou)?nt)|acc(ou)?nt.?(no|num|credit)|" +
-        "salary|billing|credit.?(rating|score)|([_-]|\\b)ccn([_-]|\\b)|" +
+        "(credit|debit|bank|visa).?(card|num|no|acc(ou)?nt)|acc(ou)?nt.?(no|num|credit)|routing.?num|"
+        + "salary|billing|beneficiary|credit.?(rating|score)|([_-]|\\b)(ccn|cvv|iban)([_-]|\\b)|" +
         // Communications - e-mail addresses, private e-mail messages, SMS text messages, chat logs, etc.
         // "e(mail|_mail)|" + // this seems too noisy
         // Health - medical conditions, insurance status, prescription records
-        "birth.?da(te|y)|da(te|y).?(of.?)?birth|" +
-        "medical|(health|care).?plan|healthkit|appointment|prescription|" +
+        "birth.?da(te|y)|da(te|y).?(of.?)?birth|gender|([_-]|\\b)sex([_-]|\\b)|" +
+        "medical|(health|care).?plan|healthkit|appointment|prescription|patient.?(id|record)|" +
         "blood.?(type|alcohol|glucose|pressure)|heart.?(rate|rhythm)|body.?(mass|fat)|" +
         "menstrua|pregnan|insulin|inhaler|" +
         // Relationships - work and family
-        "employ(er|ee)|spouse|maiden.?name" +
+        "employ(er|ee)|spouse|maiden.?name|" +
+        // Device information
+        "([_-]|\\b)ip.?addr|mac.?addr|finger.?print" +
         // ---
         ").*"
   }
@@ -144,7 +148,8 @@ module HeuristicNames {
    */
   string notSensitiveRegexp() {
     result =
-      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|certain|concert|secretar|accountant|accountab).*"
+      "(?is).*([^\\w$.-]|redact|censor|obfuscate|hash|md5|sha|random|((?<!un)(en))?(crypt|(?<!pass)code)|"
+        + "certain|concert|secretar|account(ant|ab|ing|ed)|file|path|([_-]|\\b)url).*"
   }
 
   /**