Extend PQL with new operators (!=, not like) and support for null/empty values

doc/04_Searching_For_Data_In_Index/09_Pimcore_Query_Language/README.md

@@ -15,21 +15,38 @@ FIELDNAME = IDENTIFIER{.IDENTIFIER}
 RELATION_FIELD_NAME = FIELDNAME:ENTITYNAME.FIELDNAME      
 IDENTIFIER = [a-zA-Z_]\w*
 ENTITYNAME = [a-zA-Z_]\w*
-OPERATOR = "="|"<"|">"|">="|"<="|"LIKE"
-VALUE = INTEGER | FLOAT | "'" STRING "'" | '"' STRING '"'
+OPERATOR = "="|"!="|"<"|">"|">="|"<="|"LIKE"|"NOT LIKE"
+NULL = "NULL"
+EMPTY = "EMPTY"
+VALUE = INTEGER | FLOAT | "'" STRING "'" | '"' STRING '"' | NULL | EMPTY
 QUERY_STRING_QUERY = "QUERY('" STRING "')"
 ```
 
 ### Operators
 
-| Operator | Description                                                                                                            | Examples                                     |
-|----------|------------------------------------------------------------------------------------------------------------------------|----------------------------------------------|
-| `=`        | equal                                                                                                                  | `field = "value"`                            |
-| `<`        | smaller than                                                                                                           | `field < 100`                                |
-| `<=`       | smaller or equal than                                                                                                  | `field <= 100`                               |
-| `=>`       | bigger or equal than                                                                                                   | `field >= 100`                               |
-| `>`        | bigger than                                                                                                            | `field > 100`                                |
-| `LIKE`     | equal with wildcard support<br/><em>* matches zero or more characters</em><br/><em>? matches any single character</em> | `field like "val*"`<br/>`field like "val?e"` |
+| Operator   | Description                                                                                                                               | Examples                                             |
+|------------|-------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------|
+| `=`        | equal (case-sensitive)                                                                                                                    | `field = "value"`                                    |
+| `!=`       | not equal (case-sensitive)                                                                                                                | `field != "value"`                                   |
+| `<`        | smaller than                                                                                                                              | `field < 100`                                        |
+| `<=`       | smaller or equal than                                                                                                                     | `field <= 100`                                       |
+| `=>`       | bigger or equal than                                                                                                                      | `field >= 100`                                       |
+| `>`        | bigger than                                                                                                                               | `field > 100`                                        |
+| `LIKE`     | equal with wildcard support (case-insensitive)<br/><em>* matches zero or more characters</em><br/><em>? matches any single character</em> | `field like "val*"`<br/>`field like "val?e"`         |
+| `NOT LIKE` | not equal with wildcard support (case-insensitive)<br/><em>* matches zero or more characters</em><br/><em>? matches any single character</em>                | `field not like "val*"`<br/>`field not like "val?e"` |
+
+### Null/Empty Values
+
+To search for null and empty values use the `NULL`/`EMPTY` keywords. Those can be used together with the `=` and `!=` operators to search for fields without value. Keep in mind that there can be a difference between `NULL` and an empty string. The `EMPTY` keyword is a shortcut for `NULL` or an empty string.
+
+**Examples:**
+
+```
+field = NULL
+field != NULL
+field = EMPTY # same as: field = NULL OR field = ''
+field != EMPTY # same as: field != NULL AND field != ''
+```
 
 ### AND / OR / Brackets
 
@@ -40,7 +57,7 @@ You can combine multiple conditions using the `AND` and `OR` operators. You can
 ```
 field1 = "value1" AND field2 = "value2"
 field1 = "value1" AND (field2 = "value2" OR field3 = "value3")
-(field1 = "value1" AND (field2 = "value2" OR field3 = "value3")) or field4 = "value4"
+(field1 = "value1" AND (field2 = "value2" OR field3 = "value3")) OR field4 = "value4"
 ```
 
 
@@ -98,19 +115,20 @@ The PQL allows passing OpenSearch [query string queries](https://opensearch.org/
 
 All examples are based on the `Car` data object class of the [Pimcore Demo](https://pimcore.com/en/try).
 
-| Query                                                               | Description                                                                                                               | 
-|---------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|
-| `series = "E-Type" AND (color = "green" OR productionYear < 1965)`  | All E-Type models which are green or produced before 1965.                                                                |
-| `manufacturer:Manufacturer.name = "Alfa" and productionYear > 1965` | All Alfa cars produced after 1965.                                                                                        |
-| `genericImages:Asset.fullPath like "/Car Images/vw/*"`              | All cars with a image linked in the `genericImages` image gallery which is contained in the asset folder `/Car Images/vw`. |
-| `color = "red" or color = "blue"`                                   | All red or blue cars using standard PQL syntax.                                                                           |
-| `Query("standard_fields.color:(red or blue)")`                      | All red or blue cars using simple query string syntax.                                                                    |
-
+| Query                                                               | Description                                                                                                                | 
+|---------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|
+| `series = "E-Type" AND (color = "green" OR productionYear < 1965)`  | All E-Type models which are green or produced before 1965.                                                                 |
+| `manufacturer:Manufacturer.name = "Alfa" AND productionYear > 1965` | All Alfa cars produced after 1965.                                                                                         |
+| `genericImages:Asset.fullPath LIKE "/Car Images/vw/*"`              | All cars with a image linked in the `genericImages` image gallery which is contained in the asset folder `/Car Images/vw`. |
+| `color = "red" OR color = "blue"`                                   | All red or blue cars using standard PQL syntax.                                                                            |
+| `series = empty AND color="red"`                                    | All models where the series is empty and the color is red.                                                                 |
+| `Query("standard_fields.color:(red OR blue)")`                      | All red or blue cars using simple query string syntax.                                                                     |
 
 ## Limitations
 
 * When searching for related elements the maximum possible results amount of sub queries is 65.000, see also [terms query documentation](https://opensearch.org/docs/latest/query-dsl/term/terms/).
 * Filtering for asset metadata fields is only possible if they are defined as predefined asset metadata or via the asset metadata class definitions bundle. Custom asset metadata fields directly defined on single assets are not supported.
+* Reserved keywords (`AND`, `OR`, `LIKE`, `NOT LIKE`, `NULL`, `EMPTY`) cannot be used as field names.
 
 ## Further Reading
 

src/Enum/QueryLanguage/QueryTokenType.php

@@ -22,16 +22,20 @@ enum QueryTokenType: string
     case T_INTEGER = 'T_INTEGER';
     case T_FLOAT = 'T_FLOAT';
     case T_STRING = 'T_STRING';
+    case T_NULL = 'T_NULL';
+    case T_EMPTY = 'T_EMPTY';
     case T_FIELDNAME = 'T_FIELDNAME';
     case T_RELATION_FIELD = 'T_RELATION_FIELD';
     case T_AND = 'T_AND';
     case T_OR = 'T_OR';
     case T_EQ = 'T_EQ';
+    case T_NEQ = 'T_NEQ';
     case T_GT = 'T_GT';
     case T_GTE = 'T_GTE';
     case T_LT = 'T_LT';
     case T_LTE = 'T_LTE';
     case T_LIKE = 'T_LIKE';
+    case T_NOT_LIKE = 'T_NOT_LIKE';
     case T_LPAREN = 'T_LPAREN';
     case T_RPAREN = 'T_RPAREN';
     case T_QUERY_STRING = 'T_QUERY_STRING';

src/QueryLanguage/Pql/Lexer.php

@@ -22,23 +22,16 @@
 
 /**
  * CONDITION = EXPRESSION | EXPRESSION ("AND" | "OR") EXPRESSION
- *
  * EXPRESSION = "(" CONDITION ")" | COMPARISON | QUERY_STRING_QUERY
- *
  * COMPARISON = FIELDNAME OPERATOR VALUE | RELATION_COMPARISON
- *
  * RELATION_COMPARISON = RELATION_FIELD_NAME OPERATOR VALUE
- *
  * FIELDNAME = IDENTIFIER{.IDENTIFIER}
- *
  * RELATION_FIELD_NAME = FIELDNAME:IDENTIFIER{.FIELDNAME}
- *
  * IDENTIFIER = [a-zA-Z_]\w*
- *
- * OPERATOR = "="|"<"|">"|">="|"<="|"LIKE"
- *
- * VALUE = INTEGER | FLOAT | "'" STRING "'" | '"' STRING '"'
- *
+ * OPERATOR = "="|"!="|"<"|">"|">="|"<="|"LIKE"|"NOT LIKE"
+ * NULL = "NULL"
+ * EMPTY = "EMPTY"
+ * VALUE = INTEGER | FLOAT | "'" STRING "'" | '"' STRING '"' | NULL | EMPTY
  * QUERY_STRING_QUERY = 'QUERY("' STRING '")'
  */
 class Lexer extends AbstractLexer implements LexerInterface
@@ -55,23 +48,43 @@ class Lexer extends AbstractLexer implements LexerInterface
 
     private const REGEX_STRING_DOUBLE_QUOTE = '"(?:[^"]|"")*"';
 
-    private const REGEX_OPERATOR = '>=|<=|=|>|<|like';
+    private const REGEX_EMPTY = 'null|empty';
+
+    private const REGEX_OPERATOR = '>=|<=|!=|=|>|<|not like|like';
 
     private const REGEX_PARANTHESES = '\(|\)';
 
+    private const TOKEN_MAP = [
+        '=' => QueryTokenType::T_EQ,
+        '!=' => QueryTokenType::T_NEQ,
+        '>' => QueryTokenType::T_GT,
+        '<' => QueryTokenType::T_LT,
+        '>=' => QueryTokenType::T_GTE,
+        '<=' => QueryTokenType::T_LTE,
+        'like' => QueryTokenType::T_LIKE,
+        'not like' => QueryTokenType::T_NOT_LIKE,
+        'and' => QueryTokenType::T_AND,
+        'or' => QueryTokenType::T_OR,
+        '(' => QueryTokenType::T_LPAREN,
+        ')' => QueryTokenType::T_RPAREN,
+        'null' => QueryTokenType::T_NULL,
+        'empty' => QueryTokenType::T_EMPTY,
+    ];
+
     /**
      * Lexical catchable patterns.
      */
     protected function getCatchablePatterns(): array
     {
         return [
             self::REGEX_QUERY_STRING,
-            self::REGEX_RELATION_FIELD,
-            self::REGEX_FIELD_NAME,
-            self::REGEX_NUMBERS,
             self::REGEX_STRING_SINGLE_QUOTE,
             self::REGEX_STRING_DOUBLE_QUOTE,
             self::REGEX_OPERATOR,
+            self::REGEX_EMPTY,
+            self::REGEX_RELATION_FIELD,
+            self::REGEX_FIELD_NAME,
+            self::REGEX_NUMBERS,
             self::REGEX_PARANTHESES,
         ];
     }
@@ -106,44 +119,8 @@ protected function getType(&$value): QueryTokenType
                 $typeToken = QueryTokenType::T_QUERY_STRING;
 
                 break;
-            case $value === '(':
-                $typeToken = QueryTokenType::T_LPAREN;
-
-                break;
-            case $value === ')':
-                $typeToken = QueryTokenType::T_RPAREN;
-
-                break;
-            case strtolower($value) === 'and':
-                $typeToken = QueryTokenType::T_AND;
-
-                break;
-            case strtolower($value) === 'or':
-                $typeToken = QueryTokenType::T_OR;
-
-                break;
-            case $value === '=':
-                $typeToken = QueryTokenType::T_EQ;
-
-                break;
-            case $value === '>':
-                $typeToken = QueryTokenType::T_GT;
-
-                break;
-            case $value === '<':
-                $typeToken = QueryTokenType::T_LT;
-
-                break;
-            case $value === '>=':
-                $typeToken = QueryTokenType::T_GTE;
-
-                break;
-            case $value === '<=':
-                $typeToken = QueryTokenType::T_LTE;
-
-                break;
-            case strtolower($value) === 'like':
-                $typeToken = QueryTokenType::T_LIKE;
+            case isset(self::TOKEN_MAP[strtolower($value)]):
+                $typeToken = self::TOKEN_MAP[strtolower($value)];
 
                 break;
             case preg_match('#' . self::REGEX_RELATION_FIELD . '#', $value):

src/QueryLanguage/Pql/Parser.php

@@ -38,18 +38,27 @@ final class Parser implements ParserInterface
 
     private const OPERATOR_TOKENS = [
         QueryTokenType::T_EQ,
+        QueryTokenType::T_NEQ,
         QueryTokenType::T_GT,
         QueryTokenType::T_LT,
         QueryTokenType::T_GTE,
         QueryTokenType::T_LTE,
         QueryTokenType::T_LIKE,
+        QueryTokenType::T_NOT_LIKE,
     ];
 
     private const NUMERIC_TOKENS = [
         QueryTokenType::T_INTEGER,
         QueryTokenType::T_FLOAT,
     ];
 
+    private const VALUE_TOKENS = [
+        QueryTokenType::T_STRING,
+        QueryTokenType::T_NULL,
+        QueryTokenType::T_EMPTY,
+        ...self::NUMERIC_TOKENS,
+    ];
+
     private int $index = 0;
 
     public function __construct(
@@ -166,7 +175,13 @@ private function parseComparison(array &$subQueries): array|ParseResultSubQuery
         $this->validateCurrentTokenNotEmpty();
 
         if (!$this->currentToken() || !$this->currentToken()->isA(...self::FIELD_NAME_TOKENS)) {
-            $this->throwParsingException('a field name', '`' . ($this->currentToken()['value'] ?? 'null') . '`');
+            $tokenValue = $this->currentToken()['value'] ?? 'null';
+            $message = null;
+            if (in_arrayi($tokenValue, ['and', 'or', 'like', 'not like', 'null', 'empty'])) {
+                $message = sprintf('Expected %s, found %s.', 'a field name', '`' . $tokenValue . '`')
+                    . ' Reserved keywords cannot be used as field name.';
+            }
+            $this->throwParsingException('a field name', '`' . $tokenValue . '`', $message);
         }
 
         /** @var Token $fieldToken */
@@ -187,8 +202,21 @@ private function parseComparison(array &$subQueries): array|ParseResultSubQuery
 
         // Adjusting expectation for the value type to include both strings and numerics
         $valueToken = $this->currentToken();
-        if (!$valueToken || !$valueToken->isA(QueryTokenType::T_STRING, ...self::NUMERIC_TOKENS)) {
-            $this->throwParsingException('a string or numeric value', '`' . ($valueToken['value'] ?? 'null') . '`');
+        if (!$valueToken || !$valueToken->isA(...self::VALUE_TOKENS)) {
+            $this->throwParsingException(
+                'a string, numeric value or a empty/null keyword',
+                '`' . ($valueToken['value'] ?? 'null') . '`'
+            );
+        }
+
+        if (!$operatorToken->isA(QueryTokenType::T_EQ, QueryTokenType::T_NEQ)
+            && $valueToken->isA(QueryTokenType::T_NULL, QueryTokenType::T_EMPTY)
+        ) {
+            $this->throwParsingException(
+                'a valid value',
+                '`' . ($valueToken['value'] ?? 'null') . '`',
+                'Operator `' . $operatorToken->value . '` does not support null/empty values'
+            );
         }
 
         $this->advance(); // Prepare for next
@@ -209,6 +237,12 @@ private function parseComparison(array &$subQueries): array|ParseResultSubQuery
             ? $this->stringToNumber($valueToken->value)
             : $valueToken->value;
 
+        $value = $valueToken->isA(QueryTokenType::T_NULL) ?
+            QueryTokenType::T_NULL : $value;
+
+        $value = $valueToken->isA(QueryTokenType::T_EMPTY) ?
+            QueryTokenType::T_EMPTY : $value;
+
         return $this->pqlAdapter->translateOperatorToSearchQuery($operatorTokenType, $field, $value);
     }