Add and improve sort order modifiers

config/pimcore/config.yaml

@@ -41,7 +41,17 @@ pimcore_generic_data_index:
               - lowercase
           generic_data_index_path_analyzer:
             tokenizer: generic_data_index_path_tokenizer
-
+        normalizer:
+          generic_data_index_sort_normalizer:
+            type: custom
+            filter:
+              - lowercase
+          generic_data_index_sort_truncate_normalizer:
+            type: custom
+            char_filter:
+              - generic_data_index_sort_truncate
+            filter:
+              - lowercase
         tokenizer:
           generic_data_index_ngram_tokenzier:
             type: ngram
@@ -52,7 +62,11 @@ pimcore_generic_data_index:
             type: whitespace
           generic_data_index_path_tokenizer:
             type: "path_hierarchy"
-
+        char_filter:
+          generic_data_index_sort_truncate:
+            type: pattern_replace
+            pattern: ^(.{256})(.*)$
+            replacement: $1
     system_fields_settings:
       general:
         id:
@@ -78,18 +92,27 @@ pimcore_generic_data_index:
               type: text
               analyzer: standard
               search_analyzer: generic_data_index_whitespace_analyzer
+            sort:
+              type: keyword
+              normalizer: generic_data_index_sort_normalizer
         path:
           type: text
           analyzer: generic_data_index_path_analyzer
           fields:
             keyword:
               type: keyword
+            sort:
+              type: keyword
+              normalizer: generic_data_index_sort_normalizer
         fullPath:
           type: text
           analyzer: generic_data_index_path_analyzer
           fields:
             keyword:
               type: keyword
+            sort:
+              type: keyword
+              normalizer: generic_data_index_sort_normalizer
         pathLevels:
           type: nested
           properties:
@@ -180,6 +203,10 @@ pimcore_generic_data_index:
           type: boolean
         classname:
           type: keyword
+          fields:
+            sort:
+              type: keyword
+              normalizer: generic_data_index_sort_normalizer
       asset:
         mimetype:
           type: keyword

config/services/search-index-adapter/open-search.yaml

@@ -1,5 +1,6 @@
 imports:
     - { resource: 'services/search-index-adapter/open-search/modifiers/*' }
+    - { resource: 'services/search-index-adapter/open-search/pql-field-name-transformers.yaml' }
 
 services:
     _defaults:
@@ -67,9 +68,6 @@ services:
     Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\SubQueriesProcessorInterface:
         class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\SubQueriesProcessor
 
-    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformer\:
-        resource: '../../../src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameTransformer'
-        tags: ['pimcore.generic_data_index.pql_field_name_transformer']
 
     Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameValidator\:
         resource: '../../../src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameValidator'

config/services/search-index-adapter/open-search/modifiers/search-modifiers.yaml

@@ -17,6 +17,7 @@ services:
   Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\FullTextSearch\FullTextSearchHandlers: ~
 
   Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\Sort\TreeSortHandlers: ~
+  Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\Sort\OrderByFieldsHandlers: ~
 
   Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\Search\Modifier\QueryLanguage\QueryLanguageHandlers: ~
 

config/services/search-index-adapter/open-search/pql-field-name-transformers.yaml

@@ -0,0 +1,36 @@
+services:
+    _defaults:
+        autowire: true
+        autoconfigure: true
+        public: false
+
+    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformer\AssetMetadataDefaultLanguageTransformer:
+      tags:
+        - {name: pimcore.generic_data_index.pql_field_name_transformer, priority: 9}
+        - {name: pimcore.generic_data_index.pql_field_name_transformer_sort, priority: 9}
+
+    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformer\FieldCategoryTransformer:
+      tags:
+        - {name: pimcore.generic_data_index.pql_field_name_transformer, priority: 10}
+        - {name: pimcore.generic_data_index.pql_field_name_transformer_sort, priority: 10}
+
+    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformer\IdTransformer:
+      tags:
+        - {name: pimcore.generic_data_index.pql_field_name_transformer, priority: 5}
+
+    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformer\ImageGalleryTransformer:
+      tags:
+        - {name: pimcore.generic_data_index.pql_field_name_transformer, priority: 6}
+
+    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformer\KeywordTransformer:
+      tags:
+        - {name: pimcore.generic_data_index.pql_field_name_transformer, priority: 0}
+        - {name: pimcore.generic_data_index.pql_field_name_transformer_sort, priority: 0}
+
+    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformer\RelationsTransformer:
+      tags:
+        - {name: pimcore.generic_data_index.pql_field_name_transformer, priority: 0}
+
+    Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformer\SortTransformer:
+      tags:
+        - {name: pimcore.generic_data_index.pql_field_name_transformer_sort, priority: 1}

doc/04_Searching_For_Data_In_Index/05_Search_Modifiers/README.md

@@ -52,10 +52,13 @@ $search->addModifier(new ParentIdFilter(1))
 
 ### Sort Modifiers
 
-| Modifier                                                                            | Modifier Category      | Description                                                                                                                                                                                                                                             |
-|-------------------------------------------------------------------------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [OrderByFullPath](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Sort/Tree/OrderByFullPath.php) | Tree related sorting   | Order by full path (including element key)                                                                                                                                                                                                              |
-| [OrderByPageNumber](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Sort/Tree/OrderByPageNumber.php) | Search related sorting | Use inverted search for large amounts of data (this modifier is added to the search when there are at least 1000 results by default, and page number is above the half of total pages. Furthermore, existing sorting has to be already applied.) |
+If multiple sort modifiers are added to the search, the order of the modifiers is important. The search result will be sorted by the first added modifier first, then by the second added modifier and so on.
+
+| Modifier                                                                                                                                     | Modifier Category      | Description                                                                                                                                                                                                                                                                                                                                                                         |
+|----------------------------------------------------------------------------------------------------------------------------------------------|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [OrderByFullPath](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Sort/Tree/OrderByFullPath.php)     | Tree related sorting   | Order by full path (including element key)                                                                                                                                                                                                                                                                                                                                          |
+| [OrderByField](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Sort/OrderByField.php)                | Field based sorting    | Order by given field name.<br/>If `$enablePqlFieldNameResolution` is set to true (default) [Pimcore Query Language](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Sort/OrderByField.php) field name resolution logic is enabled. Therefore it's possible to use short field names then instead of specifying the full path in OpenSearch. |
+| [OrderByPageNumber](https://github.com/pimcore/generic-data-index-bundle/blob/1.x/src/Model/Search/Modifier/Sort/Tree/OrderByPageNumber.php) | Search related sorting | Use inverted search for large amounts of data (this modifier is added to the search when there are at least 1000 results by default, and page number is above the half of total pages. Furthermore, existing sorting has to be already applied.)                                                                                                                                    |
 
 ### Aggregations
 

src/Enum/DependencyInjection/ServiceTag.php

@@ -30,5 +30,6 @@ enum ServiceTag: string
     case DOCUMENT_TYPE_SERIALIZATION_HANDLER = 'pimcore.generic_data_index.document_type_serialization_handler';
     case ASSET_MAPPING_PROVIDER = 'pimcore.generic_data_index.asset.mapping_provider';
     case PQL_FIELD_NAME_TRANSFORMER = 'pimcore.generic_data_index.pql_field_name_transformer';
+    case PQL_FIELD_NAME_TRANSFORMER_SORT = 'pimcore.generic_data_index.pql_field_name_transformer_sort';
     case PQL_FIELD_NAME_VALIDATOR = 'pimcore.generic_data_index.pql_field_name_validator';
 }

src/Model/Search/Modifier/Sort/OrderByField.php

@@ -0,0 +1,45 @@
+<?php
+declare(strict_types=1);
+
+/**
+ * Pimcore
+ *
+ * This source file is available under two different licenses:
+ * - GNU General Public License version 3 (GPLv3)
+ * - Pimcore Commercial License (PCL)
+ * Full copyright and license information is available in
+ * LICENSE.md which is distributed with this source code.
+ *
+ *  @copyright  Copyright (c) Pimcore GmbH (http://www.pimcore.org)
+ *  @license    http://www.pimcore.org/license     GPLv3 and PCL
+ */
+
+namespace Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Sort;
+
+use Pimcore\Bundle\GenericDataIndexBundle\Enum\Search\SortDirection;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\SearchModifierInterface;
+
+final readonly class OrderByField implements SearchModifierInterface
+{
+    public function __construct(
+        private string $fieldName,
+        private SortDirection $direction = SortDirection::ASC,
+        private bool $enablePqlFieldNameResolution = true,
+    ) {
+    }
+
+    public function getFieldName(): string
+    {
+        return $this->fieldName;
+    }
+
+    public function getDirection(): SortDirection
+    {
+        return $this->direction;
+    }
+
+    public function isPqlFieldNameResolutionEnabled(): bool
+    {
+        return $this->enablePqlFieldNameResolution;
+    }
+}

src/SearchIndexAdapter/OpenSearch/Asset/FieldDefinitionAdapter/KeywordAdapter.php

@@ -29,7 +29,14 @@ public function getIndexMapping(): array
     {
         return [
             'type' => AttributeType::KEYWORD->value,
-            'ignore_above' => 10000,
+            'ignore_above' => 8191,
+            'fields' => [
+                'sort' => [
+                    'type' => AttributeType::KEYWORD->value,
+                    'ignore_above' => 8191,
+                    'normalizer' => 'generic_data_index_sort_truncate_normalizer',
+                ],
+            ],
         ];
     }
 

src/SearchIndexAdapter/OpenSearch/IndexMappingService.php

@@ -65,6 +65,11 @@ public function getMappingForTextKeyword(array $attributes): array
                         'type' => AttributeType::KEYWORD->value,
                         'ignore_above' => 1024,
                     ],
+                    'sort' => [
+                        'type' => AttributeType::KEYWORD->value,
+                        'ignore_above' => 8191,
+                        'normalizer' => 'generic_data_index_sort_truncate_normalizer',
+                    ],
                 ]
             ),
         ];

src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameTransformer/AssetMetadataDefaultLanguageTransformer.php

@@ -21,14 +21,12 @@
 use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\MappingProperty;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\MappingAnalyzerServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformerInterface;
-use Symfony\Component\DependencyInjection\Attribute\AsTaggedItem;
 
 /**
  * Used by asset metadata fields to add the default language to the field name.
  *
  * @internal
  */
-#[AsTaggedItem(priority: 9)]
 final readonly class AssetMetadataDefaultLanguageTransformer implements FieldNameTransformerInterface
 {
     public function __construct(

src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameTransformer/FieldCategoryTransformer.php

@@ -20,15 +20,13 @@
 use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\IndexEntity;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\MappingAnalyzerServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformerInterface;
-use Symfony\Component\DependencyInjection\Attribute\AsTaggedItem;
 
 /**
  * Prepends the field category (system_fields, standard_fields, custom_fields)
  * to the field name if it is not already present.
  *
  * @internal
  */
-#[AsTaggedItem(priority: 10)]
 final readonly class FieldCategoryTransformer implements FieldNameTransformerInterface
 {
     public function __construct(

src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameTransformer/IdTransformer.php

@@ -19,14 +19,12 @@
 use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\IndexEntity;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\MappingAnalyzerServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformerInterface;
-use Symfony\Component\DependencyInjection\Attribute\AsTaggedItem;
 
 /**
  * Used for image and video data types to transform the field name to the id field.
  *
  * @internal
  */
-#[AsTaggedItem(priority: 5)]
 final readonly class IdTransformer implements FieldNameTransformerInterface
 {
     public function __construct(

src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameTransformer/ImageGalleryTransformer.php

@@ -20,14 +20,12 @@
 use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\IndexEntity;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\MappingAnalyzerServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformerInterface;
-use Symfony\Component\DependencyInjection\Attribute\AsTaggedItem;
 
 /**
  * Used for the image gallery data type.
  *
  * @internal
  */
-#[AsTaggedItem(priority: 6)]
 final readonly class ImageGalleryTransformer implements FieldNameTransformerInterface
 {
     public function __construct(

src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameTransformer/KeywordTransformer.php

@@ -19,14 +19,12 @@
 use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\IndexEntity;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\MappingAnalyzerServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformerInterface;
-use Symfony\Component\DependencyInjection\Attribute\AsTaggedItem;
 
 /**
  * Used to filter based on a keyword subfield if available.
  *
  * @internal
  */
-#[AsTaggedItem(priority: 0)]
 final readonly class KeywordTransformer implements FieldNameTransformerInterface
 {
     public function __construct(

src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameTransformer/RelationsTransformer.php

@@ -20,14 +20,12 @@
 use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\IndexEntity;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\MappingAnalyzerServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformerInterface;
-use Symfony\Component\DependencyInjection\Attribute\AsTaggedItem;
 
 /**
  * Used by relation data types to transform the field name to the correct element type relation field.
  *
  * @internal
  */
-#[AsTaggedItem(priority: 0)]
 final readonly class RelationsTransformer implements FieldNameTransformerInterface
 {
     public function __construct(

src/SearchIndexAdapter/OpenSearch/QueryLanguage/FieldNameTransformer/SortTransformer.php

@@ -0,0 +1,53 @@
+<?php
+declare(strict_types=1);
+
+/**
+ * Pimcore
+ *
+ * This source file is available under two different licenses:
+ * - GNU General Public License version 3 (GPLv3)
+ * - Pimcore Commercial License (PCL)
+ * Full copyright and license information is available in
+ * LICENSE.md which is distributed with this source code.
+ *
+ *  @copyright  Copyright (c) Pimcore GmbH (http://www.pimcore.org)
+ *  @license    http://www.pimcore.org/license     GPLv3 and PCL
+ */
+
+namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformer;
+
+use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\IndexEntity;
+use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\MappingAnalyzerServiceInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\OpenSearch\QueryLanguage\FieldNameTransformerInterface;
+
+/**
+ * Used to filter based on a sort subfield if available.
+ *
+ * @internal
+ */
+final readonly class SortTransformer implements FieldNameTransformerInterface
+{
+    public function __construct(
+        private MappingAnalyzerServiceInterface $mappingAnalyzerService
+    ) {
+    }
+
+    public function transformFieldName(string $fieldName, array $indexMapping, ?IndexEntity $targetEntity): ?string
+    {
+        if (!$this->mappingAnalyzerService->fieldPathExists($fieldName, $indexMapping)) {
+            return null;
+        }
+
+        $fullFieldName = $fieldName . '.sort';
+        if ($this->mappingAnalyzerService->fieldPathExists($fullFieldName, $indexMapping)) {
+            return $fullFieldName;
+        }
+
+        return null;
+    }
+
+    public function stopPropagation(): bool
+    {
+        return true;
+    }
+}